01HNS5HTEBQA3FMZRYQYDT8034: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HNS5JFXJ22PFEGZPYWGXYKNT

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-04T13:42:14.339Zjob dependencies complete; ready to run (waiting for 9 h 21 m 6 s)
22024-02-04T13:59:18.439Zjob assigned to worker 01HNT6KP77K91DR3C8YEZQE4PG (queued for 17 m 4 s)
32024-02-04T13:59:19.204Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-04T13:59:29.300Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-04T13:59:29.300Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-04T13:59:32.157Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-04T13:59:32.157Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-04T13:59:36.952Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-04T13:59:36.952Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-04T13:59:43.226Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-04T13:59:43.229Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-04T13:59:43.255Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-04T13:59:43.255Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-04T13:59:43.284Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-04T13:59:43.284Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-04T13:59:43.299Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-04T13:59:43.299Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-04T13:59:43.325Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-04T13:59:43.325Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-04T13:59:43.351Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-04T13:59:43.351Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-04T13:59:43.377Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-04T13:59:43.377Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-04T13:59:43.397Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-04T13:59:43.400Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-04T13:59:56.732Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-04T13:59:56.732Zstarting task 0: "setup"
282024-02-04T13:59:56.776Z++ uname -s
292024-02-04T13:59:56.779Z+ kern=SunOS
302024-02-04T13:59:56.779Z+ case "$kern" in
312024-02-04T13:59:56.779Z+ groupadd -g 12345 build
322024-02-04T13:59:56.784Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-04T13:59:58.827Z+ zfs create -o mountpoint=/work rpool/work
342024-02-04T13:59:58.917Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-04T13:59:58.920Z+ home_fs=zfs
362024-02-04T13:59:58.920Z+ [[ zfs == autofs ]]
372024-02-04T13:59:58.920Z+ mkdir -p /home/build
382024-02-04T13:59:58.923Z+ chown build:build /home/build /work
392024-02-04T14:00:00.926Z+ chmod 0700 /home/build /work
402024-02-04T14:00:00.929Zprocess exited: duration 4214 ms, exit code 0
 
412024-02-04T14:00:00.938Zstarting task 1: "authentication"
422024-02-04T14:00:00.960Zprocess exited: duration 24 ms, exit code 0
 
432024-02-04T14:00:00.969Zstarting task 2: "build"
442024-02-04T14:00:00.973Z+ banner cores
452024-02-04T14:00:00.997Z
462024-02-04T14:00:00.997Z #### #### ##### ###### ####
472024-02-04T14:00:00.997Z # # # # # # # #
482024-02-04T14:00:00.997Z # # # # # ##### ####
492024-02-04T14:00:00.997Z # # # ##### # #
502024-02-04T14:00:00.997Z # # # # # # # # #
512024-02-04T14:00:00.997Z #### #### # # ###### ####
522024-02-04T14:00:00.997Z
532024-02-04T14:00:01.001Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-04T14:00:01.009Z+ echo 'input bins dir contains:'
552024-02-04T14:00:01.012Zinput bins dir contains:
562024-02-04T14:00:01.012Z+ ls -ltr /input/build/work/bins
572024-02-04T14:00:01.012Ztotal 760364
582024-02-04T14:00:01.012Z-rw-r--r-- 1 root root 121402564 Feb 4 13:59 crucible-downstairs.gz
592024-02-04T14:00:01.012Z-rw-r--r-- 1 root root 90892491 Feb 4 13:59 crucible-hammer.gz
602024-02-04T14:00:01.012Z-rw-r--r-- 1 root root 104770823 Feb 4 13:59 crutest.gz
612024-02-04T14:00:01.012Z-rw-r--r-- 1 root root 71991054 Feb 4 13:59 dsc.gz
622024-02-04T14:00:01.012Z+ echo 'input script dir contains:'
632024-02-04T14:00:01.012Zinput script dir contains:
642024-02-04T14:00:01.015Z+ ls -ltr /input/build/work/scripts
652024-02-04T14:00:01.015Ztotal 38
662024-02-04T14:00:01.015Z-rw-r--r-- 1 root root 3056 Feb 4 13:59 perf-downstairs-tick.d
672024-02-04T14:00:01.015Z-rw-r--r-- 1 root root 939 Feb 4 13:59 test_ds.sh
682024-02-04T14:00:01.015Z-rw-r--r-- 1 root root 3746 Feb 4 13:59 test_live_repair.sh
692024-02-04T14:00:01.015Z-rw-r--r-- 1 root root 6416 Feb 4 13:59 test_repair.sh
702024-02-04T14:00:01.015Z-rw-r--r-- 1 root root 2969 Feb 4 13:59 test_replay.sh
712024-02-04T14:00:01.016Z-rw-r--r-- 1 root root 9465 Feb 4 13:59 test_up.sh
722024-02-04T14:00:01.016Z-rw-r--r-- 1 root root 2824 Feb 4 13:59 upstairs_info.d
732024-02-04T14:00:01.016Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-04T14:00:01.018Z+ echo ' chmod input script dir contains:'
752024-02-04T14:00:01.019Z chmod input script dir contains:
762024-02-04T14:00:01.019Z+ ls -ltr /input/build/work/scripts
772024-02-04T14:00:01.023Ztotal 38
782024-02-04T14:00:01.023Z-rwxr-xr-x 1 root root 3056 Feb 4 13:59 perf-downstairs-tick.d
792024-02-04T14:00:01.023Z-rwxr-xr-x 1 root root 939 Feb 4 13:59 test_ds.sh
802024-02-04T14:00:01.023Z-rwxr-xr-x 1 root root 3746 Feb 4 13:59 test_live_repair.sh
812024-02-04T14:00:01.023Z-rwxr-xr-x 1 root root 6416 Feb 4 13:59 test_repair.sh
822024-02-04T14:00:01.023Z-rwxr-xr-x 1 root root 2969 Feb 4 13:59 test_replay.sh
832024-02-04T14:00:01.023Z-rwxr-xr-x 1 root root 9465 Feb 4 13:59 test_up.sh
842024-02-04T14:00:01.023Z-rwxr-xr-x 1 root root 2824 Feb 4 13:59 upstairs_info.d
852024-02-04T14:00:01.023Z+ banner unpack
862024-02-04T14:00:01.026Z
872024-02-04T14:00:01.026Z # # # # ##### ## #### # #
882024-02-04T14:00:01.026Z # # ## # # # # # # # # #
892024-02-04T14:00:01.026Z # # # # # # # # # # ####
902024-02-04T14:00:01.027Z # # # # # ##### ###### # # #
912024-02-04T14:00:01.027Z # # # ## # # # # # # #
922024-02-04T14:00:01.027Z #### # # # # # #### # #
932024-02-04T14:00:01.027Z
942024-02-04T14:00:01.027Z+ mkdir -p /var/tmp/bins
952024-02-04T14:00:01.027Z+ for t in "$input/bins/"*.gz
962024-02-04T14:00:01.030Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-04T14:00:01.030Z+ b=crucible-downstairs.gz
982024-02-04T14:00:01.030Z+ b=crucible-downstairs
992024-02-04T14:00:01.030Z+ gunzip
1002024-02-04T14:00:05.299Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-04T14:00:05.303Z+ for t in "$input/bins/"*.gz
1022024-02-04T14:00:05.304Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-04T14:00:05.308Z+ b=crucible-hammer.gz
1042024-02-04T14:00:05.308Z+ b=crucible-hammer
1052024-02-04T14:00:05.308Z+ gunzip
1062024-02-04T14:00:08.578Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-04T14:00:08.582Z+ for t in "$input/bins/"*.gz
1082024-02-04T14:00:08.582Z++ basename /input/build/work/bins/crutest.gz
1092024-02-04T14:00:08.585Z+ b=crutest.gz
1102024-02-04T14:00:08.585Z+ b=crutest
1112024-02-04T14:00:08.585Z+ gunzip
1122024-02-04T14:00:12.190Z+ chmod +x /var/tmp/bins/crutest
1132024-02-04T14:00:12.193Z+ for t in "$input/bins/"*.gz
1142024-02-04T14:00:12.196Z++ basename /input/build/work/bins/dsc.gz
1152024-02-04T14:00:12.196Z+ b=dsc.gz
1162024-02-04T14:00:12.196Z+ b=dsc
1172024-02-04T14:00:12.196Z+ gunzip
1182024-02-04T14:00:14.774Z+ chmod +x /var/tmp/bins/dsc
1192024-02-04T14:00:14.777Z+ export BINDIR=/var/tmp/bins
1202024-02-04T14:00:14.777Z+ BINDIR=/var/tmp/bins
1212024-02-04T14:00:14.777Z+ banner repair
1222024-02-04T14:00:14.780Z
1232024-02-04T14:00:14.780Z ##### ###### ##### ## # #####
1242024-02-04T14:00:14.780Z # # # # # # # # # #
1252024-02-04T14:00:14.781Z # # ##### # # # # # # #
1262024-02-04T14:00:14.781Z ##### # ##### ###### # #####
1272024-02-04T14:00:14.781Z # # # # # # # # #
1282024-02-04T14:00:14.781Z # # ###### # # # # # #
1292024-02-04T14:00:14.781Z
1302024-02-04T14:00:14.781Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-04T14:00:14.920ZTurn off color for downstairs dump
1322024-02-04T14:00:14.955ZCreating region directory at: /var/tmp/test_repair
1332024-02-04T14:00:14.958ZCreating dsc directory at: /tmp/dsc
1342024-02-04T14:00:15.334ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.3850628
1352024-02-04T14:00:15.402ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.06879366
1362024-02-04T14:00:15.467ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.06454067
1372024-02-04T14:00:15.467ZRegion set with 3 regions was created
1382024-02-04T14:00:15.486Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1131 1132 1133
1392024-02-04T14:00:15.486ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-04T14:00:15.545Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.577246996Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135}
1412024-02-04T14:00:15.549Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.578461488Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135}
1422024-02-04T14:00:15.549Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.578493453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135}
1432024-02-04T14:00:15.549Z{"msg":"Crucible 54ffa644-1acf-4088-bb36-bebd90c5e322 has session id: 1ee8b08a-297d-4475-90da-32123681c866","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.578515518Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1442024-02-04T14:00:15.549Z{"msg":"Upstairs opts: Upstairs UUID: 54ffa644-1acf-4088-bb36-bebd90c5e322, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.578545698Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1452024-02-04T14:00:15.549Z{"msg":"Crucible stats registered with UUID: 54ffa644-1acf-4088-bb36-bebd90c5e322","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.578569917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1462024-02-04T14:00:15.549ZCrucible runtime is spawned
1472024-02-04T14:00:15.550Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.579688189Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135}
1482024-02-04T14:00:15.550Z{"msg":"54ffa644-1acf-4088-bb36-bebd90c5e322 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.580078353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1492024-02-04T14:00:15.550Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.580133859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1502024-02-04T14:00:15.550Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.580160536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1512024-02-04T14:00:15.550Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.580225683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1522024-02-04T14:00:15.550Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.58026912Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1532024-02-04T14:00:15.550Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.580336836Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1542024-02-04T14:00:15.550Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.580413426Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1552024-02-04T14:00:15.550Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.58061516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1562024-02-04T14:00:15.550Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.580733921Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1572024-02-04T14:00:15.550Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.580783095Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1582024-02-04T14:00:15.550Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.581390938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1592024-02-04T14:00:15.551Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.581486326Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1602024-02-04T14:00:15.551Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.581530538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1612024-02-04T14:00:15.554Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.584924322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1622024-02-04T14:00:15.555Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.585006053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1632024-02-04T14:00:15.555Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.585053623Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1642024-02-04T14:00:15.555Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.585195264Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1652024-02-04T14:00:15.555Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.585278023Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1662024-02-04T14:00:15.555Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.585350616Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1672024-02-04T14:00:15.555Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.587635291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1682024-02-04T14:00:15.555Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.587701143Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1692024-02-04T14:00:15.555Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.587793021Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1702024-02-04T14:00:15.560Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.589466962Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1712024-02-04T14:00:15.560Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.589539373Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1722024-02-04T14:00:15.560Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.589588716Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1732024-02-04T14:00:15.560Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.589633577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1742024-02-04T14:00:15.560Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.589730651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1752024-02-04T14:00:15.560Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.589807054Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1762024-02-04T14:00:15.560Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.589879988Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1772024-02-04T14:00:15.560Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.589954593Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1782024-02-04T14:00:15.560Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590052826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1792024-02-04T14:00:15.560Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590134246Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1802024-02-04T14:00:15.560Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590267756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1812024-02-04T14:00:15.560Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590331159Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1822024-02-04T14:00:15.561Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590358309Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1832024-02-04T14:00:15.561Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590418977Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1842024-02-04T14:00:15.561Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590478492Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1852024-02-04T14:00:15.561Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590537356Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1862024-02-04T14:00:15.561Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590566005Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1872024-02-04T14:00:15.561Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590630949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
1882024-02-04T14:00:15.561Z,"":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1892024-02-04T14:00:15.561Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590695705Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1902024-02-04T14:00:15.561Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590728901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1912024-02-04T14:00:15.561Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590753224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1922024-02-04T14:00:15.561Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590811367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1932024-02-04T14:00:15.561Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590836057Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1942024-02-04T14:00:15.562Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590865309Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1952024-02-04T14:00:15.562Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590918582Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1962024-02-04T14:00:15.562Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590944115Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1972024-02-04T14:00:15.562Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.590998948Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1982024-02-04T14:00:15.562Z{"msg":"54ffa644-1acf-4088-bb36-bebd90c5e322 is now active with session: 1ee8b08a-297d-4475-90da-32123681c866","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.591030949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
1992024-02-04T14:00:15.562Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.591052469Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135,"session_id":"1ee8b08a-297d-4475-90da-32123681c866"}
2002024-02-04T14:00:15.562Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:14.591096693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135}
2012024-02-04T14:00:15.562ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-04T14:00:15.562ZFill test
2032024-02-04T14:00:15.917ZRead and Verify all blocks (0..600 range:false)
2042024-02-04T14:00:16.024ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-04T14:00:16.024ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-04T14:00:16.027Z----------------------------------------------------------------
2072024-02-04T14:00:16.027Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-04T14:00:16.027ZStates: Active Active Active
2092024-02-04T14:00:16.028ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-04T14:00:16.028Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-04T14:00:16.028Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-04T14:00:16.028Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-04T14:00:16.028Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-04T14:00:16.028Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-04T14:00:16.028Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-04T14:00:16.028Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-04T14:00:16.028Z New 0 0 0 0
2182024-02-04T14:00:16.028Z Sent 0 0 0 0
2192024-02-04T14:00:16.028Z Done 6 6 6 18
2202024-02-04T14:00:16.028Z Skipped 0 0 0 0
2212024-02-04T14:00:16.029Z Error 0 0 0 0
2222024-02-04T14:00:16.029ZLast Flush: 1006 1006 1006
2232024-02-04T14:00:16.029ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-04T14:00:16.029ZUpstairs last five completed: 13 12 11 10 9
2252024-02-04T14:00:16.029ZCLIENT: Up:0 ds:6 act:3
2262024-02-04T14:00:20.025Z----------------------------------------------------------------
2272024-02-04T14:00:20.025Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-04T14:00:20.025ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-04T14:00:20.025ZUpstairs last five completed: 14 13 12 11 10
2302024-02-04T14:00:20.029ZCLIENT: Up:0 ds:0 act:3
2312024-02-04T14:00:20.029ZCLIENT: All crucible jobs finished, exiting program
2322024-02-04T14:00:20.029Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:00:19.059404998Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1135}
2332024-02-04T14:00:20.035Z
2342024-02-04T14:00:20.035ZBegin loop 1
2352024-02-04T14:00:20.035ZDownstairs to restart: 2
2362024-02-04T14:00:20.064Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.097636217Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137}
2372024-02-04T14:00:20.064Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098113707Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137}
2382024-02-04T14:00:20.064Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098148949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137}
2392024-02-04T14:00:20.069ZCrucible runtime is spawned
2402024-02-04T14:00:20.069Z{"msg":"Crucible 8ccb4a70-2556-4e4f-ab66-4cb45e0a85ea has session id: c2207d68-b821-44d2-9f3a-eb9b89675045","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098412907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2412024-02-04T14:00:20.069Z{"msg":"Upstairs opts: Upstairs UUID: 8ccb4a70-2556-4e4f-ab66-4cb45e0a85ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098519336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2422024-02-04T14:00:20.069Z{"msg":"Crucible stats registered with UUID: 8ccb4a70-2556-4e4f-ab66-4cb45e0a85ea","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098595846Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2432024-02-04T14:00:20.069Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098628973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137}
2442024-02-04T14:00:20.069Z{"msg":"8ccb4a70-2556-4e4f-ab66-4cb45e0a85ea active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098689242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2452024-02-04T14:00:20.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098800721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2462024-02-04T14:00:20.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.09882836Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2472024-02-04T14:00:20.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098895647Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2482024-02-04T14:00:20.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098930127Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2492024-02-04T14:00:20.070Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.098995497Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2502024-02-04T14:00:20.070Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.099040103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2512024-02-04T14:00:20.070Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.099135124Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2522024-02-04T14:00:20.070Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.099456169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2532024-02-04T14:00:20.070Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.099507522Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2542024-02-04T14:00:20.070Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.099569659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2552024-02-04T14:00:20.070Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.099597111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2562024-02-04T14:00:20.070Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.099630514Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2572024-02-04T14:00:20.070Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.10043109Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2582024-02-04T14:00:20.070Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.100555517Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2592024-02-04T14:00:20.070Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.100646378Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2602024-02-04T14:00:20.070Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.100760044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2612024-02-04T14:00:20.070Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.101323401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2622024-02-04T14:00:20.070Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.101571504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2632024-02-04T14:00:20.070Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.101808163Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2642024-02-04T14:00:20.070Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.101865207Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2652024-02-04T14:00:20.074Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.102264432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2662024-02-04T14:00:20.074Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.102352642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2672024-02-04T14:00:20.074Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.10245575Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2682024-02-04T14:00:20.075Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.102499146Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2692024-02-04T14:00:20.075Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.102523447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2702024-02-04T14:00:20.075Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.102545639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2712024-02-04T14:00:20.075Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.102823094Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2722024-02-04T14:00:20.075Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.10355846Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2732024-02-04T14:00:20.075Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.103680668Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2742024-02-04T14:00:20.075Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.103787673Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2752024-02-04T14:00:20.075Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.103814594Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2762024-02-04T14:00:20.075ZWait for a query_work_queue command to finish before sending IO
2772024-02-04T14:00:20.075Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.103843686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2782024-02-04T14:00:20.075Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.103868606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2792024-02-04T14:00:20.075Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.103911764Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2802024-02-04T14:00:20.075Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.103937244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2812024-02-04T14:00:20.075Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.103978997Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2822024-02-04T14:00:20.075Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104003052Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2832024-02-04T14:00:20.075Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104048565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2842024-02-04T14:00:20.075Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104072146Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2852024-02-04T14:00:20.075Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104119412Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2862024-02-04T14:00:20.075Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104159624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2872024-02-04T14:00:20.075Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104214844Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2882024-02-04T14:00:20.075Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104288062Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2892024-02-04T14:00:20.075Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104314504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2902024-02-04T14:00:20.075Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104378537Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2912024-02-04T14:00:20.076Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.10440611Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2922024-02-04T14:00:20.076Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104473269Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2932024-02-04T14:00:20.076Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104499975Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2942024-02-04T14:00:20.076Z{"msg":"8ccb4a70-2556-4e4f-ab66-4cb45e0a85ea is now active with session: c2207d68-b821-44d2-9f3a-eb9b89675045","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104563885Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2952024-02-04T14:00:20.076ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2962024-02-04T14:00:20.076Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104646484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137,"session_id":"c2207d68-b821-44d2-9f3a-eb9b89675045"}
2972024-02-04T14:00:20.076Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:19.104707387Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137}
2982024-02-04T14:00:20.076ZLoading write count information from file "/tmp/test_repair_verify.data"
2992024-02-04T14:00:20.076ZRun Repair workload
3002024-02-04T14:00:20.076Z01/30 Write block 379 len 8192 data: 2 2
3012024-02-04T14:00:20.079Z02/30 Read block 218 len 4096
3022024-02-04T14:00:20.080Z03/30 Read block 111 len 32768
3032024-02-04T14:00:20.080Z04/30 Read block 441 len 32768
3042024-02-04T14:00:20.082Z05/30 Write block 65 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3052024-02-04T14:00:20.082Z06/30 Read block 165 len 32768
3062024-02-04T14:00:20.085Z07/30 Read block 279 len 40960
3072024-02-04T14:00:20.088Z08/30 Write block 262 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3082024-02-04T14:00:20.088Z09/30 Read block 271 len 8192
3092024-02-04T14:00:20.088Z10/30 Read block 327 len 24576
3102024-02-04T14:00:20.091Z11/30 Write block 411 len 20480 data: 2 2 2 2 2
3112024-02-04T14:00:20.091Z12/30 Flush
3122024-02-04T14:00:20.120Z13/30 Write block 14 len 28672 data: 2 2 2 2 2 2 2
3132024-02-04T14:00:20.123Z14/30 Read block 403 len 16384
3142024-02-04T14:00:20.126Z15/30 Write block 560 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3152024-02-04T14:00:20.126Z16/30 Write block 556 len 36864 data: 2 2 2 2 3 3 3 3 3
3162024-02-04T14:00:20.126Z17/30 Flush
3172024-02-04T14:00:20.161Z18/30 Write block 461 len 20480 data: 2 2 2 2 2
3182024-02-04T14:00:20.164Z19/30 Flush
3192024-02-04T14:00:20.170Z20/30 Write block 266 len 16384 data: 3 3 3 3
3202024-02-04T14:00:20.170Z21/30 Flush
3212024-02-04T14:00:20.173Z22/30 Write block 511 len 4096 data: 2
3222024-02-04T14:00:20.173Z23/30 Read block 361 len 12288
3232024-02-04T14:00:20.173Z24/30 Write block 334 len 28672 data: 2 2 2 2 2 2 2
3242024-02-04T14:00:20.176Z25/30 Write block 59 len 16384 data: 2 2 2 2
3252024-02-04T14:00:20.176Z26/30 Write block 167 len 36864 data: 2 2 2 2 2 2 2 2 2
3262024-02-04T14:00:20.176Z27/30 Read block 329 len 36864
3272024-02-04T14:00:20.179Z28/30 Write block 118 len 4096 data: 2
3282024-02-04T14:00:20.179Z29/30 Read block 511 len 4096
3292024-02-04T14:00:20.183Z30/30 Write block 398 len 12288 data: 2 2 2
3302024-02-04T14:00:20.183Z----------------------------------------------------------------
3312024-02-04T14:00:20.183Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3322024-02-04T14:00:20.183ZStates: Active Active Active
3332024-02-04T14:00:20.183ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3342024-02-04T14:00:20.183Z 1 Acked 1000 Write 2 Done Done Done false
3352024-02-04T14:00:20.183Z 2 Acked 1001 Read 1 Done Done Sent false
3362024-02-04T14:00:20.183Z 3 Acked 1002 Read 8 Done Done Sent false
3372024-02-04T14:00:20.183Z 4 Acked 1003 Read 8 Done Done Sent false
3382024-02-04T14:00:20.183Z 5 Acked 1004 Write 10 Done Done Sent false
3392024-02-04T14:00:20.183Z 6 Acked 1005 Read 8 Done Done Sent false
3402024-02-04T14:00:20.183Z 7 Acked 1006 Read 10 Done Done Sent false
3412024-02-04T14:00:20.183Z 8 Acked 1007 Write 10 Done Done Sent false
3422024-02-04T14:00:20.183Z 9 Acked 1008 Read 2 Done Done Sent false
3432024-02-04T14:00:20.183Z 10 Acked 1009 Read 6 Done Done Sent false
3442024-02-04T14:00:20.183Z 11 Acked 1010 Write 5 Done Done Sent false
3452024-02-04T14:00:20.184Z 12 Acked 1011 Flush 0 Done Done Sent false
3462024-02-04T14:00:20.184Z 13 Acked 1012 Write 7 Done Done Sent false
3472024-02-04T14:00:20.184Z 14 Acked 1013 Read 4 Done Done Sent false
3482024-02-04T14:00:20.184Z 15 Acked 1014 Write 10 Done Done Sent false
3492024-02-04T14:00:20.184Z 16 Acked 1015 Write 9 Done Done Sent false
3502024-02-04T14:00:20.184Z 17 Acked 1016 Flush 0 Done Done Sent false
3512024-02-04T14:00:20.184Z 18 Acked 1017 Write 5 Done Done Sent false
3522024-02-04T14:00:20.184Z 19 Acked 1018 Flush 0 Done Done Sent false
3532024-02-04T14:00:20.184Z 20 Acked 1019 Write 4 Done Done Sent false
3542024-02-04T14:00:20.184Z 21 Acked 1020 Flush 0 Done Done Sent false
3552024-02-04T14:00:20.184Z 22 Acked 1021 Write 1 Done Done Sent false
3562024-02-04T14:00:20.184Z 23 Acked 1022 Read 3 Done Done Sent false
3572024-02-04T14:00:20.184Z 24 Acked 1023 Write 7 Done Done Sent false
3582024-02-04T14:00:20.184Z 25 Acked 1024 Write 4 Done Done Sent false
3592024-02-04T14:00:20.184Z 26 Acked 1025 Write 9 Done Done Sent false
3602024-02-04T14:00:20.184Z 27 Acked 1026 Read 9 Done Done Sent false
3612024-02-04T14:00:20.184Z 28 Acked 1027 Write 1 Done Done Sent false
3622024-02-04T14:00:20.184Z 29 Acked 1028 Read 1 Done Done Sent false
3632024-02-04T14:00:20.184Z 30 Acked 1029 Write 3 Sent Sent Sent false
3642024-02-04T14:00:20.184Z STATES DS:0 DS:1 DS:2 TOTAL
3652024-02-04T14:00:20.184Z New 0 0 0 0
3662024-02-04T14:00:20.184Z Sent 1 1 29 31
3672024-02-04T14:00:20.184Z Done 29 29 1 59
3682024-02-04T14:00:20.184Z Skipped 0 0 0 0
3692024-02-04T14:00:20.184Z Error 0 0 0 0
3702024-02-04T14:00:20.185ZLast Flush: 1020 1020 0
3712024-02-04T14:00:20.185ZDownstairs last five completed:
3722024-02-04T14:00:20.185ZUpstairs last five completed: 30 29 28 27 26
3732024-02-04T14:00:20.185Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:00:19.21577243Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1137}
3742024-02-04T14:00:20.185ZWrote out file "/tmp/test_repair_verify.data"
3752024-02-04T14:00:20.189Z
3762024-02-04T14:00:20.195ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3772024-02-04T14:00:20.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3782024-02-04T14:00:20.225Z 0 000-019 2 2 1 3 3 1 F F F <---
3792024-02-04T14:00:20.225Z 1 020-039 2 2 1 3 3 1 F F F <---
3802024-02-04T14:00:20.225Z 2 040-059 1 1 1 1 1 1 T T F <---
3812024-02-04T14:00:20.225Z 3 060-079 2 2 1 2 2 1 T T F <---
3822024-02-04T14:00:20.228Z 4 080-099 1 1 1 1 1 1 F F F
3832024-02-04T14:00:20.228Z 5 100-119 1 1 1 1 1 1 T T F <---
3842024-02-04T14:00:20.228Z 6 120-139 1 1 1 1 1 1 F F F
3852024-02-04T14:00:20.229Z 7 140-159 1 1 1 1 1 1 F F F
3862024-02-04T14:00:20.229Z 8 160-179 1 1 1 1 1 1 T T F <---
3872024-02-04T14:00:20.229Z 9 180-199 1 1 1 1 1 1 F F F
3882024-02-04T14:00:20.229Z 10 200-219 1 1 1 1 1 1 F F F
3892024-02-04T14:00:20.229Z 11 220-239 1 1 1 1 1 1 F F F
3902024-02-04T14:00:20.229Z 12 240-259 1 1 1 1 1 1 F F F
3912024-02-04T14:00:20.229Z 13 260-279 2 2 1 5 5 1 F F F <---
3922024-02-04T14:00:20.229Z 14 280-299 1 1 1 1 1 1 F F F
3932024-02-04T14:00:20.229Z 15 300-319 1 1 1 1 1 1 F F F
3942024-02-04T14:00:20.229Z 16 320-339 1 1 1 1 1 1 T T F <---
3952024-02-04T14:00:20.229Z 17 340-359 1 1 1 1 1 1 T T F <---
3962024-02-04T14:00:20.229Z 18 360-379 2 2 1 2 2 1 F F T <---
3972024-02-04T14:00:20.229Z 19 380-399 2 2 1 2 2 1 T T T <---
3982024-02-04T14:00:20.229Z 20 400-419 2 2 1 2 2 1 T T F <---
3992024-02-04T14:00:20.229Z 21 420-439 1 1 1 1 1 1 F F F
4002024-02-04T14:00:20.229Z 22 440-459 1 1 1 1 1 1 F F F
4012024-02-04T14:00:20.229Z 23 460-479 2 2 1 4 4 1 F F F <---
4022024-02-04T14:00:20.229Z 24 480-499 1 1 1 1 1 1 F F F
4032024-02-04T14:00:20.229Z 25 500-519 1 1 1 1 1 1 T T F <---
4042024-02-04T14:00:20.229Z 26 520-539 1 1 1 1 1 1 F F F
4052024-02-04T14:00:20.229Z 27 540-559 2 2 1 3 3 1 F F F <---
4062024-02-04T14:00:20.229Z 28 560-579 2 2 1 3 3 1 F F F <---
4072024-02-04T14:00:20.229Z 29 580-599 1 1 1 1 1 1 F F F
4082024-02-04T14:00:20.229ZMax gen: 2, Max flush: 5
4092024-02-04T14:00:20.229ZError: Difference in extent metadata found!
4102024-02-04T14:00:20.233ZA Difference in extent metadata is expected here
4112024-02-04T14:00:20.233ZOn loop 1
4122024-02-04T14:00:20.233Z
4132024-02-04T14:00:20.236ZVerifying data now
4142024-02-04T14:00:27.526ZNo change after verify
4152024-02-04T14:00:27.526ZLoop: 1 Downstairs dump after verify (and repair):
4162024-02-04T14:00:27.545ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4172024-02-04T14:00:27.545Z 0 000-019 2 2 2 3 3 3 F F F
4182024-02-04T14:00:27.545Z 1 020-039 2 2 2 3 3 3 F F F
4192024-02-04T14:00:27.546Z 2 040-059 3 3 3 6 6 6 F F F
4202024-02-04T14:00:27.546Z 3 060-079 3 3 3 6 6 6 F F F
4212024-02-04T14:00:27.546Z 4 080-099 1 1 1 1 1 1 F F F
4222024-02-04T14:00:27.546Z 5 100-119 3 3 3 6 6 6 F F F
4232024-02-04T14:00:27.546Z 6 120-139 1 1 1 1 1 1 F F F
4242024-02-04T14:00:27.546Z 7 140-159 1 1 1 1 1 1 F F F
4252024-02-04T14:00:27.546Z 8 160-179 3 3 3 6 6 6 F F F
4262024-02-04T14:00:27.546Z 9 180-199 1 1 1 1 1 1 F F F
4272024-02-04T14:00:27.546Z 10 200-219 1 1 1 1 1 1 F F F
4282024-02-04T14:00:27.546Z 11 220-239 1 1 1 1 1 1 F F F
4292024-02-04T14:00:27.546Z 12 240-259 1 1 1 1 1 1 F F F
4302024-02-04T14:00:27.546Z 13 260-279 2 2 2 5 5 5 F F F
4312024-02-04T14:00:27.546Z 14 280-299 1 1 1 1 1 1 F F F
4322024-02-04T14:00:27.550Z 15 300-319 1 1 1 1 1 1 F F F
4332024-02-04T14:00:27.550Z 16 320-339 3 3 3 6 6 6 F F F
4342024-02-04T14:00:27.550Z 17 340-359 3 3 3 6 6 6 F F F
4352024-02-04T14:00:27.550Z 18 360-379 2 2 2 2 2 2 F F F
4362024-02-04T14:00:27.550Z 19 380-399 3 3 3 6 6 6 F F F
4372024-02-04T14:00:27.550Z 20 400-419 3 3 3 6 6 6 F F F
4382024-02-04T14:00:27.550Z 21 420-439 1 1 1 1 1 1 F F F
4392024-02-04T14:00:27.550Z 22 440-459 1 1 1 1 1 1 F F F
4402024-02-04T14:00:27.550Z 23 460-479 2 2 2 4 4 4 F F F
4412024-02-04T14:00:27.550Z 24 480-499 1 1 1 1 1 1 F F F
4422024-02-04T14:00:27.550Z 25 500-519 3 3 3 6 6 6 F F F
4432024-02-04T14:00:27.550Z 26 520-539 1 1 1 1 1 1 F F F
4442024-02-04T14:00:27.550Z 27 540-559 2 2 2 3 3 3 F F F
4452024-02-04T14:00:27.550Z 28 560-579 2 2 2 3 3 3 F F F
4462024-02-04T14:00:27.550Z 29 580-599 1 1 1 1 1 1 F F F
4472024-02-04T14:00:27.550ZMax gen: 3, Max flush: 6
4482024-02-04T14:00:27.553Z
4492024-02-04T14:00:27.553ZBegin loop 2
4502024-02-04T14:00:27.553ZDownstairs to restart: 0
4512024-02-04T14:00:27.577Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.610671255Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145}
4522024-02-04T14:00:27.577Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.611113633Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145}
4532024-02-04T14:00:27.581Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.611194459Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145}
4542024-02-04T14:00:27.581Z{"msg":"Crucible 0b9ab2c0-44f8-4297-8c73-fd26cf746a70 has session id: 34d5774c-51fe-402a-9e85-585c2e44f40f","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.611237256Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4552024-02-04T14:00:27.581Z{"msg":"Upstairs opts: Upstairs UUID: 0b9ab2c0-44f8-4297-8c73-fd26cf746a70, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.611322761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4562024-02-04T14:00:27.581Z{"msg":"Crucible stats registered with UUID: 0b9ab2c0-44f8-4297-8c73-fd26cf746a70","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.611350958Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4572024-02-04T14:00:27.581ZCrucible runtime is spawned
4582024-02-04T14:00:27.581Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.61158933Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145}
4592024-02-04T14:00:27.581Z{"msg":"0b9ab2c0-44f8-4297-8c73-fd26cf746a70 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.611838037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4602024-02-04T14:00:27.581Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.611888652Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4612024-02-04T14:00:27.581Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.611984367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4622024-02-04T14:00:27.581Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.612050051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4632024-02-04T14:00:27.581Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.612206601Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4642024-02-04T14:00:27.581Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.61236661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4652024-02-04T14:00:27.581Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.612621062Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4662024-02-04T14:00:27.581Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.612823041Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4672024-02-04T14:00:27.581Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.612893917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4682024-02-04T14:00:27.581Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.612955666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4692024-02-04T14:00:27.581Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.612980849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4702024-02-04T14:00:27.581Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.613004148Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4712024-02-04T14:00:27.581Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.613036583Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4722024-02-04T14:00:27.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.613844622Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4732024-02-04T14:00:27.581Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.613907413Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4742024-02-04T14:00:27.581Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.613936721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4752024-02-04T14:00:27.581Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.613976331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4762024-02-04T14:00:27.585Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.614654116Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4772024-02-04T14:00:27.586Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.61473119Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4782024-02-04T14:00:27.586Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.615153305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4792024-02-04T14:00:27.586Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.615360844Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4802024-02-04T14:00:27.586Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.61581502Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4812024-02-04T14:00:27.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616144122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4822024-02-04T14:00:27.586Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616202061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4832024-02-04T14:00:27.586Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.61627829Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4842024-02-04T14:00:27.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616348997Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4852024-02-04T14:00:27.586Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616421648Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4862024-02-04T14:00:27.586Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616515368Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4872024-02-04T14:00:27.586Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616674Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4882024-02-04T14:00:27.586Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616844656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4892024-02-04T14:00:27.586Z{"msg":"[0]R flush_numbers[0..12]: [3, 3, 6, 6, 1, 6, 1, 1, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616907475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4902024-02-04T14:00:27.586Z{"msg":"[0]R generation[0..12]: [2, 2, 3, 3, 1, 3, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.616932581Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4912024-02-04T14:00:27.586Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
4922024-02-04T14:00:27.586Z,"level":30,"time":"2024-02-04T14:00:26.616983002Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4932024-02-04T14:00:27.586Z{"msg":"[1]R flush_numbers[0..12]: [3, 3, 6, 6, 1, 6, 1, 1, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617008134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4942024-02-04T14:00:27.586Z{"msg":"[1]R generation[0..12]: [2, 2, 3, 3, 1, 3, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617051655Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4952024-02-04T14:00:27.586Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.6170768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4962024-02-04T14:00:27.586Z{"msg":"[2]R flush_numbers[0..12]: [3, 3, 6, 6, 1, 6, 1, 1, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617100304Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4972024-02-04T14:00:27.586Z{"msg":"[2]R generation[0..12]: [2, 2, 3, 3, 1, 3, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617141203Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4982024-02-04T14:00:27.586Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.61716637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
4992024-02-04T14:00:27.586Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617190435Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5002024-02-04T14:00:27.586Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617235721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5012024-02-04T14:00:27.587Z{"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617258686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5022024-02-04T14:00:27.587Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617298964Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5032024-02-04T14:00:27.587Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617322615Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5042024-02-04T14:00:27.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617361076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5052024-02-04T14:00:27.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617385065Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5062024-02-04T14:00:27.587Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617425658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5072024-02-04T14:00:27.587Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.61744903Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5082024-02-04T14:00:27.587Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617488211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5092024-02-04T14:00:27.587Z{"msg":"0b9ab2c0-44f8-4297-8c73-fd26cf746a70 is now active with session: 34d5774c-51fe-402a-9e85-585c2e44f40f","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617510917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5102024-02-04T14:00:27.587Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617530969Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145,"session_id":"34d5774c-51fe-402a-9e85-585c2e44f40f"}
5112024-02-04T14:00:27.587Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:26.617551834Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145}
5122024-02-04T14:00:27.587ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5132024-02-04T14:00:27.587ZLoading write count information from file "/tmp/test_repair_verify.data"
5142024-02-04T14:00:27.587ZRun Repair workload
5152024-02-04T14:00:27.587Z01/30 Write block 60 len 24576 data: 3 3 3 2 2 3
5162024-02-04T14:00:27.590Z02/30 Read block 393 len 20480
5172024-02-04T14:00:27.590Z03/30 Read block 413 len 12288
5182024-02-04T14:00:27.593Z04/30 Write block 391 len 16384 data: 2 2 2 2
5192024-02-04T14:00:27.593Z05/30 Read block 305 len 12288
5202024-02-04T14:00:27.594Z06/30 Read block 179 len 28672
5212024-02-04T14:00:27.596Z07/30 Read block 333 len 20480
5222024-02-04T14:00:27.596Z08/30 Read block 443 len 24576
5232024-02-04T14:00:27.599Z09/30 Write block 177 len 36864 data: 2 2 2 2 2 2 2 2 2
5242024-02-04T14:00:27.599Z10/30 Read block 294 len 40960
5252024-02-04T14:00:27.602Z11/30 Flush
5262024-02-04T14:00:27.604Z12/30 Write block 389 len 20480 data: 2 2 3 3 3
5272024-02-04T14:00:27.607Z13/30 Write block 249 len 8192 data: 2 2
5282024-02-04T14:00:27.607Z14/30 Read block 393 len 4096
5292024-02-04T14:00:27.607Z15/30 Read block 472 len 12288
5302024-02-04T14:00:27.610Z16/30 Write block 243 len 40960 data: 2 2 2 2 2 2 3 3 2 2
5312024-02-04T14:00:27.610Z17/30 Read block 211 len 24576
5322024-02-04T14:00:27.613Z18/30 Flush
5332024-02-04T14:00:27.638Z19/30 Write block 556 len 24576 data: 3 3 3 3 4 4
5342024-02-04T14:00:27.641Z20/30 Write block 258 len 28672 data: 2 2 2 2 3 3 3
5352024-02-04T14:00:27.641Z21/30 Write block 308 len 28672 data: 2 2 2 2 2 2 2
5362024-02-04T14:00:27.644Z22/30 Write block 261 len 4096 data: 3
5372024-02-04T14:00:27.644Z23/30 Read block 93 len 32768
5382024-02-04T14:00:27.644Z24/30 Read block 333 len 16384
5392024-02-04T14:00:27.647Z25/30 Read block 531 len 20480
5402024-02-04T14:00:27.647Z26/30 Read block 394 len 4096
5412024-02-04T14:00:27.650Z27/30 Write block 199 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5422024-02-04T14:00:27.650Z28/30 Write block 262 len 40960 data: 4 4 4 3 4 4 4 4 3 3
5432024-02-04T14:00:27.653Z29/30 Write block 113 len 20480 data: 2 2 2 2 2
5442024-02-04T14:00:27.653Z30/30 Read block 174 len 8192
5452024-02-04T14:00:27.656Z----------------------------------------------------------------
5462024-02-04T14:00:27.656Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5472024-02-04T14:00:27.656ZStates: Active Active Active
5482024-02-04T14:00:27.656ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5492024-02-04T14:00:27.656Z 1 Acked 1000 Write 6 Done Done Done false
5502024-02-04T14:00:27.656Z 2 Acked 1001 Read 5 Done Done Done false
5512024-02-04T14:00:27.656Z 3 Acked 1002 Read 3 Sent Done Done false
5522024-02-04T14:00:27.656Z 4 Acked 1003 Write 4 Sent Done Done false
5532024-02-04T14:00:27.656Z 5 Acked 1004 Read 3 Sent Done Done false
5542024-02-04T14:00:27.656Z 6 Acked 1005 Read 7 Sent Done Done false
5552024-02-04T14:00:27.656Z 7 Acked 1006 Read 5 Sent Done Done false
5562024-02-04T14:00:27.656Z 8 Acked 1007 Read 6 Sent Done Done false
5572024-02-04T14:00:27.656Z 9 Acked 1008 Write 9 Sent Done Done false
5582024-02-04T14:00:27.656Z 10 Acked 1009 Read 10 Sent Done Done false
5592024-02-04T14:00:27.657Z 11 Acked 1010 Flush 0 Sent Done Done false
5602024-02-04T14:00:27.657Z 12 Acked 1011 Write 5 Sent Done Done false
5612024-02-04T14:00:27.657Z 13 Acked 1012 Write 2 Sent Done Done false
5622024-02-04T14:00:27.657Z 14 Acked 1013 Read 1 Sent Done Done false
5632024-02-04T14:00:27.657Z 15 Acked 1014 Read 3 Sent Done Done false
5642024-02-04T14:00:27.657Z 16 Acked 1015 Write 10 Sent Done Done false
5652024-02-04T14:00:27.657Z 17 Acked 1016 Read 6 Sent Done Done false
5662024-02-04T14:00:27.657Z 18 Acked 1017 Flush 0 Sent Done Done false
5672024-02-04T14:00:27.657Z 19 Acked 1018 Write 6 Sent Done Done false
5682024-02-04T14:00:27.657Z 20 Acked 1019 Write 7 Sent Done Done false
5692024-02-04T14:00:27.657Z 21 Acked 1020 Write 7 Sent Done Done false
5702024-02-04T14:00:27.657Z 22 Acked 1021 Write 1 Sent Done Done false
5712024-02-04T14:00:27.657Z 23 Acked 1022 Read 8 Sent Done Done false
5722024-02-04T14:00:27.657Z 24 Acked 1023 Read 4 Sent Done Done false
5732024-02-04T14:00:27.657Z 25 Acked 1024 Read 5 Sent Done Done false
5742024-02-04T14:00:27.657Z 26 Acked 1025 Read 1 Sent Done Done false
5752024-02-04T14:00:27.657Z 27 Acked 1026 Write 10 Sent Done Done false
5762024-02-04T14:00:27.657Z 28 Acked 1027 Write 10 Sent Done Done false
5772024-02-04T14:00:27.657Z 29 Acked 1028 Write 5 Sent Done Done false
5782024-02-04T14:00:27.657Z 30 Acked 1029 Read 2 Sent Sent Done false
5792024-02-04T14:00:27.657Z STATES DS:0 DS:1 DS:2 TOTAL
5802024-02-04T14:00:27.657Z New 0 0 0 0
5812024-02-04T14:00:27.657Z Sent 28 1 0 29
5822024-02-04T14:00:27.657Z Done 2 29 30 61
5832024-02-04T14:00:27.657Z Skipped 0 0 0 0
5842024-02-04T14:00:27.658Z Error 0 0 0 0
5852024-02-04T14:00:27.658ZLast Flush: 0 1017 1017
5862024-02-04T14:00:27.658ZDownstairs last five completed:
5872024-02-04T14:00:27.658ZUpstairs last five completed: 30 29 28 27 26
5882024-02-04T14:00:27.658Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:00:26.687804845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1145}
5892024-02-04T14:00:27.658ZWrote out file "/tmp/test_repair_verify.data"
5902024-02-04T14:00:27.661Z
5912024-02-04T14:00:27.665ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5922024-02-04T14:00:27.691ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5932024-02-04T14:00:27.691Z 0 000-019 2 2 2 3 3 3 F F F
5942024-02-04T14:00:27.691Z 1 020-039 2 2 2 3 3 3 F F F
5952024-02-04T14:00:27.692Z 2 040-059 3 3 3 6 6 6 F F F
5962024-02-04T14:00:27.692Z 3 060-079 3 4 4 6 7 7 T F F <---
5972024-02-04T14:00:27.692Z 4 080-099 1 1 1 1 1 1 F F F
5982024-02-04T14:00:27.692Z 5 100-119 3 3 3 6 6 6 F T T <---
5992024-02-04T14:00:27.692Z 6 120-139 1 1 1 1 1 1 F F F
6002024-02-04T14:00:27.692Z 7 140-159 1 1 1 1 1 1 F F F
6012024-02-04T14:00:27.692Z 8 160-179 3 4 4 6 7 7 F F F <---
6022024-02-04T14:00:27.692Z 9 180-199 1 4 4 1 7 7 F T T <---
6032024-02-04T14:00:27.695Z 10 200-219 1 1 1 1 1 1 F T T <---
6042024-02-04T14:00:27.695Z 11 220-239 1 1 1 1 1 1 F F F
6052024-02-04T14:00:27.695Z 12 240-259 1 4 4 1 8 8 F T T <---
6062024-02-04T14:00:27.695Z 13 260-279 2 2 2 5 5 5 F T T <---
6072024-02-04T14:00:27.695Z 14 280-299 1 1 1 1 1 1 F F F
6082024-02-04T14:00:27.695Z 15 300-319 1 1 1 1 1 1 F T T <---
6092024-02-04T14:00:27.695Z 16 320-339 3 3 3 6 6 6 F F F
6102024-02-04T14:00:27.695Z 17 340-359 3 3 3 6 6 6 F F F
6112024-02-04T14:00:27.695Z 18 360-379 2 2 2 2 2 2 F F F
6122024-02-04T14:00:27.695Z 19 380-399 3 4 4 6 8 8 F F F <---
6132024-02-04T14:00:27.695Z 20 400-419 3 3 3 6 6 6 F F F
6142024-02-04T14:00:27.695Z 21 420-439 1 1 1 1 1 1 F F F
6152024-02-04T14:00:27.695Z 22 440-459 1 1 1 1 1 1 F F F
6162024-02-04T14:00:27.695Z 23 460-479 2 2 2 4 4 4 F F F
6172024-02-04T14:00:27.695Z 24 480-499 1 1 1 1 1 1 F F F
6182024-02-04T14:00:27.695Z 25 500-519 3 3 3 6 6 6 F F F
6192024-02-04T14:00:27.695Z 26 520-539 1 1 1 1 1 1 F F F
6202024-02-04T14:00:27.695Z 27 540-559 2 2 2 3 3 3 F T T <---
6212024-02-04T14:00:27.695Z 28 560-579 2 2 2 3 3 3 F T T <---
6222024-02-04T14:00:27.695Z 29 580-599 1 1 1 1 1 1 F F F
6232024-02-04T14:00:27.695ZMax gen: 4, Max flush: 8
6242024-02-04T14:00:27.696ZError: Difference in extent metadata found!
6252024-02-04T14:00:27.698ZA Difference in extent metadata is expected here
6262024-02-04T14:00:27.699ZOn loop 2
6272024-02-04T14:00:27.699Z
6282024-02-04T14:00:27.701ZVerifying data now
6292024-02-04T14:00:33.480ZNo change after verify
6302024-02-04T14:00:33.480ZLoop: 2 Downstairs dump after verify (and repair):
6312024-02-04T14:00:33.503ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6322024-02-04T14:00:33.503Z 0 000-019 2 2 2 3 3 3 F F F
6332024-02-04T14:00:33.503Z 1 020-039 2 2 2 3 3 3 F F F
6342024-02-04T14:00:33.503Z 2 040-059 3 3 3 6 6 6 F F F
6352024-02-04T14:00:33.503Z 3 060-079 4 4 4 7 7 7 F F F
6362024-02-04T14:00:33.503Z 4 080-099 1 1 1 1 1 1 F F F
6372024-02-04T14:00:33.503Z 5 100-119 5 5 5 9 9 9 F F F
6382024-02-04T14:00:33.503Z 6 120-139 1 1 1 1 1 1 F F F
6392024-02-04T14:00:33.503Z 7 140-159 1 1 1 1 1 1 F F F
6402024-02-04T14:00:33.503Z 8 160-179 4 4 4 7 7 7 F F F
6412024-02-04T14:00:33.507Z 9 180-199 5 5 5 9 9 9 F F F
6422024-02-04T14:00:33.507Z 10 200-219 5 5 5 9 9 9 F F F
6432024-02-04T14:00:33.507Z 11 220-239 1 1 1 1 1 1 F F F
6442024-02-04T14:00:33.507Z 12 240-259 5 5 5 9 9 9 F F F
6452024-02-04T14:00:33.507Z 13 260-279 5 5 5 9 9 9 F F F
6462024-02-04T14:00:33.507Z 14 280-299 1 1 1 1 1 1 F F F
6472024-02-04T14:00:33.507Z 15 300-319 5 5 5 9 9 9 F F F
6482024-02-04T14:00:33.507Z 16 320-339 3 3 3 6 6 6 F F F
6492024-02-04T14:00:33.507Z 17 340-359 3 3 3 6 6 6 F F F
6502024-02-04T14:00:33.507Z 18 360-379 2 2 2 2 2 2 F F F
6512024-02-04T14:00:33.507Z 19 380-399 4 4 4 8 8 8 F F F
6522024-02-04T14:00:33.507Z 20 400-419 3 3 3 6 6 6 F F F
6532024-02-04T14:00:33.507Z 21 420-439 1 1 1 1 1 1 F F F
6542024-02-04T14:00:33.507Z 22 440-459 1 1 1 1 1 1 F F F
6552024-02-04T14:00:33.507Z 23 460-479 2 2 2 4 4 4 F F F
6562024-02-04T14:00:33.507Z 24 480-499 1 1 1 1 1 1 F F F
6572024-02-04T14:00:33.507Z 25 500-519 3 3 3 6 6 6 F F F
6582024-02-04T14:00:33.507Z 26 520-539 1 1 1 1 1 1 F F F
6592024-02-04T14:00:33.507Z 27 540-559 5 5 5 9 9 9 F F F
6602024-02-04T14:00:33.507Z 28 560-579 5 5 5 9 9 9 F F F
6612024-02-04T14:00:33.507Z 29 580-599 1 1 1 1 1 1 F F F
6622024-02-04T14:00:33.507ZMax gen: 5, Max flush: 9
6632024-02-04T14:00:33.511Z
6642024-02-04T14:00:33.511ZBegin loop 3
6652024-02-04T14:00:33.511ZDownstairs to restart: 1
6662024-02-04T14:00:33.539Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.572211602Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153}
6672024-02-04T14:00:33.539Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57270224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153}
6682024-02-04T14:00:33.542Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
6692024-02-04T14:00:33.542Z,"time":"2024-02-04T14:00:32.572787609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153}
6702024-02-04T14:00:33.542Z{"msg":"Crucible 4875a656-f3e9-4b84-bac6-4dc46cefc4d0 has session id: ea24b6c9-49ef-420f-a5f1-294633a429a8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.572949082Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6712024-02-04T14:00:33.542Z{"msg":"Upstairs opts: Upstairs UUID: 4875a656-f3e9-4b84-bac6-4dc46cefc4d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573070557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6722024-02-04T14:00:33.543Z{"msg":"Crucible stats registered with UUID: 4875a656-f3e9-4b84-bac6-4dc46cefc4d0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573105039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6732024-02-04T14:00:33.543Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573220827Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153}
6742024-02-04T14:00:33.543Z{"msg":"4875a656-f3e9-4b84-bac6-4dc46cefc4d0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573255036Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6752024-02-04T14:00:33.543Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573348059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6762024-02-04T14:00:33.543Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573429452Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6772024-02-04T14:00:33.543Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573470093Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6782024-02-04T14:00:33.543Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57351379Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6792024-02-04T14:00:33.543Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573649515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6802024-02-04T14:00:33.543Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.5737226Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6812024-02-04T14:00:33.543Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573812336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6822024-02-04T14:00:33.543Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573887769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6832024-02-04T14:00:33.543Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.573935726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6842024-02-04T14:00:33.543Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.574000479Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6852024-02-04T14:00:33.543Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.574043166Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6862024-02-04T14:00:33.543Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57408668Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6872024-02-04T14:00:33.543Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.574977165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6882024-02-04T14:00:33.543Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57504658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6892024-02-04T14:00:33.543Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.575253629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6902024-02-04T14:00:33.543Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.575304901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6912024-02-04T14:00:33.543Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.575979336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6922024-02-04T14:00:33.547Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.576238959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6932024-02-04T14:00:33.548Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.576920863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6942024-02-04T14:00:33.548Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.576993475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6952024-02-04T14:00:33.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.577205595Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6962024-02-04T14:00:33.548Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.577270735Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6972024-02-04T14:00:33.548Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.577303622Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6982024-02-04T14:00:33.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.577359325Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
6992024-02-04T14:00:33.548Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.577385939Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7002024-02-04T14:00:33.548Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.577449268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7012024-02-04T14:00:33.548Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.577880108Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7022024-02-04T14:00:33.548Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.578576328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7032024-02-04T14:00:33.548Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.578665448Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7042024-02-04T14:00:33.548Z{"msg":"[0]R flush_numbers[0..12]: [3, 3, 6, 7, 1, 9, 1, 1, 7, 9, 9, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.578743863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7052024-02-04T14:00:33.548Z{"msg":"[0]R generation[0..12]: [2, 2, 3, 4, 1, 5, 1, 1, 4, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.578817291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7062024-02-04T14:00:33.548Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.578894103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7072024-02-04T14:00:33.548Z{"msg":"Wait for a query_work_queue command to finish before sending IO
7082024-02-04T14:00:33.548Z[1]R flush_numbers[0..12]: [3, 3, 6, 7, 1, 9, 1, 1, 7, 9, 9, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.578999571Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7092024-02-04T14:00:33.548Z{"msg":"[1]R generation[0..12]: [2, 2, 3, 4, 1, 5, 1, 1, 4, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579034193Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7102024-02-04T14:00:33.548Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579107681Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7112024-02-04T14:00:33.548Z{"msg":"[2]R flush_numbers[0..12]: [3, 3, 6, 7, 1, 9, 1, 1, 7, 9, 9, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579156958Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7122024-02-04T14:00:33.548Z{"msg":"[2]R generation[0..12]: [2, 2, 3, 4, 1, 5, 1, 1, 4, 5, 5, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579225979Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7132024-02-04T14:00:33.548Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57929994Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7142024-02-04T14:00:33.549Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57937284Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7152024-02-04T14:00:33.549Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579447393Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7162024-02-04T14:00:33.549Z{"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57948311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7172024-02-04T14:00:33.549Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579535161Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7182024-02-04T14:00:33.549Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579559819Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7192024-02-04T14:00:33.549Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579603133Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7202024-02-04T14:00:33.549Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57962675Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7212024-02-04T14:00:33.549Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579674201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7222024-02-04T14:00:33.549Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579698331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7232024-02-04T14:00:33.549Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579738805Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7242024-02-04T14:00:33.549Z{"msg":"4875a656-f3e9-4b84-bac6-4dc46cefc4d0 is now active with session: ea24b6c9-49ef-420f-a5f1-294633a429a8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.57976256Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7252024-02-04T14:00:33.549Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7262024-02-04T14:00:33.549Z,"level":30,"time":"2024-02-04T14:00:32.579861992Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153,"session_id":"ea24b6c9-49ef-420f-a5f1-294633a429a8"}
7272024-02-04T14:00:33.549Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:32.579910817Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153}
7282024-02-04T14:00:33.549ZLoading write count information from file "/tmp/test_repair_verify.data"
7292024-02-04T14:00:33.549ZRun Repair workload
7302024-02-04T14:00:33.549Z01/30 Flush
7312024-02-04T14:00:33.552Z02/30 Flush
7322024-02-04T14:00:33.553Z03/30 Write block 392 len 4096 data: 4
7332024-02-04T14:00:33.553Z04/30 Read block 591 len 16384
7342024-02-04T14:00:33.553Z05/30 Write block 251 len 36864 data: 3 3 2 2 2 2 2 3 3
7352024-02-04T14:00:33.555Z06/30 Write block 526 len 4096 data: 2
7362024-02-04T14:00:33.556Z07/30 Write block 186 len 32768 data: 2 2 2 2 2 2 2 2
7372024-02-04T14:00:33.556Z08/30 Read block 260 len 4096
7382024-02-04T14:00:33.558Z09/30 Write block 593 len 12288 data: 2 2 2
7392024-02-04T14:00:33.558Z10/30 Read block 337 len 36864
7402024-02-04T14:00:33.561Z11/30 Flush
7412024-02-04T14:00:33.588Z12/30 Write block 454 len 32768 data: 2 2 2 2 2 2 2 3
7422024-02-04T14:00:33.591Z13/30 Read block 384 len 8192
7432024-02-04T14:00:33.591Z14/30 Read block 334 len 16384
7442024-02-04T14:00:33.594Z15/30 Read block 419 len 28672
7452024-02-04T14:00:33.594Z16/30 Read block 261 len 8192
7462024-02-04T14:00:33.594Z17/30 Read block 317 len 8192
7472024-02-04T14:00:33.597Z18/30 Write block 125 len 12288 data: 2 2 2
7482024-02-04T14:00:33.597Z19/30 Write block 317 len 32768 data: 2 2 2 2 2 2 2 2
7492024-02-04T14:00:33.600Z20/30 Write block 104 len 40960 data: 2 2 2 2 2 2 2 2 2 3
7502024-02-04T14:00:33.600Z21/30 Write block 172 len 40960 data: 3 3 3 3 2 3 3 3 3 3
7512024-02-04T14:00:33.603Z22/30 Write block 448 len 8192 data: 2 2
7522024-02-04T14:00:33.603Z23/30 Write block 305 len 24576 data: 2 2 2 3 3 3
7532024-02-04T14:00:33.603Z24/30 Read block 522 len 16384
7542024-02-04T14:00:33.609Z25/30 Read block 567 len 4096
7552024-02-04T14:00:33.609Z26/30 Flush
7562024-02-04T14:00:33.611Z27/30 Write block 74 len 36864 data: 3 2 2 2 2 2 2 2 2
7572024-02-04T14:00:33.611Z28/30 Read block 60 len 24576
7582024-02-04T14:00:33.614Z29/30 Write block 297 len 12288 data: 2 2 2
7592024-02-04T14:00:33.614Z30/30 Write block 133 len 4096 data: 2
7602024-02-04T14:00:33.618Z----------------------------------------------------------------
7612024-02-04T14:00:33.618Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:29
7622024-02-04T14:00:33.618ZStates: Active Active Active
7632024-02-04T14:00:33.618ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7642024-02-04T14:00:33.618Z 2 Acked 1001 Flush 0 Done Sent Done false
7652024-02-04T14:00:33.618Z 3 Acked 1002 Write 1 Done Sent Done false
7662024-02-04T14:00:33.618Z 4 Acked 1003 Read 4 Done Sent Done false
7672024-02-04T14:00:33.618Z 5 Acked 1004 Write 9 Done Sent Done false
7682024-02-04T14:00:33.618Z 6 Acked 1005 Write 1 Done Sent Done false
7692024-02-04T14:00:33.618Z 7 Acked 1006 Write 8 Done Sent Done false
7702024-02-04T14:00:33.618Z 8 Acked 1007 Read 1 Done Sent Done false
7712024-02-04T14:00:33.618Z 9 Acked 1008 Write 3 Done Sent Done false
7722024-02-04T14:00:33.618Z 10 Acked 1009 Read 9 Done Sent Done false
7732024-02-04T14:00:33.618Z 11 Acked 1010 Flush 0 Done Sent Done false
7742024-02-04T14:00:33.618Z 12 Acked 1011 Write 8 Done Sent Done false
7752024-02-04T14:00:33.618Z 13 Acked 1012 Read 2 Done Sent Done false
7762024-02-04T14:00:33.618Z 14 Acked 1013 Read 4 Done Sent Done false
7772024-02-04T14:00:33.618Z 15 Acked 1014 Read 7 Done Sent Done false
7782024-02-04T14:00:33.618Z 16 Acked 1015 Read 2 Done Sent Done false
7792024-02-04T14:00:33.618Z 17 Acked 1016 Read 2 Done Sent Done false
7802024-02-04T14:00:33.618Z 18 Acked 1017 Write 3 Done Sent Done false
7812024-02-04T14:00:33.618Z 19 Acked 1018 Write 8 Done Sent Done false
7822024-02-04T14:00:33.618Z 20 Acked 1019 Write 10 Done Sent Done false
7832024-02-04T14:00:33.619Z 21 Acked 1020 Write 10 Done Sent Done false
7842024-02-04T14:00:33.619Z 22 Acked 1021 Write 2 Done Sent Done false
7852024-02-04T14:00:33.619Z 23 Acked 1022 Write 6 Done Sent Done false
7862024-02-04T14:00:33.619Z 24 Acked 1023 Read 4 Done Sent Done false
7872024-02-04T14:00:33.619Z 25 Acked 1024 Read 1 Done Sent Done false
7882024-02-04T14:00:33.619Z 26 Acked 1025 Flush 0 Done Sent Done false
7892024-02-04T14:00:33.619Z 27 Acked 1026 Write 9 Done Sent Done false
7902024-02-04T14:00:33.619Z 28 Acked 1027 Read 6 Done Sent Done false
7912024-02-04T14:00:33.619Z 29 Acked 1028 Write 3 Sent Sent Sent false
7922024-02-04T14:00:33.619Z 30 Acked 1029 Write 1 Sent Sent Sent false
7932024-02-04T14:00:33.619Z STATES DS:0 DS:1 DS:2 TOTAL
7942024-02-04T14:00:33.619Z New 0 0 0 0
7952024-02-04T14:00:33.619Z Sent 2 29 2 33
7962024-02-04T14:00:33.619Z Done 27 0 27 54
7972024-02-04T14:00:33.619Z Skipped 0 0 0 0
7982024-02-04T14:00:33.619Z Error 0 0 0 0
7992024-02-04T14:00:33.619ZLast Flush: 1025 1000 1025
8002024-02-04T14:00:33.619ZDownstairs last five completed: 1000
8012024-02-04T14:00:33.619ZUpstairs last five completed: 30 29 28 27 26
8022024-02-04T14:00:33.619Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:00:32.649895382Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1153}
8032024-02-04T14:00:33.619ZWrote out file "/tmp/test_repair_verify.data"
8042024-02-04T14:00:33.623Z
8052024-02-04T14:00:33.629ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8062024-02-04T14:00:33.654ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8072024-02-04T14:00:33.654Z 0 000-019 2 2 2 3 3 3 F F F
8082024-02-04T14:00:33.655Z 1 020-039 2 2 2 3 3 3 F F F
8092024-02-04T14:00:33.655Z 2 040-059 3 3 3 6 6 6 F F F
8102024-02-04T14:00:33.655Z 3 060-079 4 4 4 7 7 7 T F T <---
8112024-02-04T14:00:33.658Z 4 080-099 1 1 1 1 1 1 T F T <---
8122024-02-04T14:00:33.658Z 5 100-119 6 5 6 13 9 13 F F F <---
8132024-02-04T14:00:33.658Z 6 120-139 6 1 6 13 1 13 T F T <---
8142024-02-04T14:00:33.658Z 7 140-159 1 1 1 1 1 1 F F F
8152024-02-04T14:00:33.658Z 8 160-179 6 4 6 13 7 13 F F F <---
8162024-02-04T14:00:33.658Z 9 180-199 6 5 6 13 9 13 F F F <---
8172024-02-04T14:00:33.658Z 10 200-219 5 5 5 9 9 9 F F F
8182024-02-04T14:00:33.658Z 11 220-239 1 1 1 1 1 1 F F F
8192024-02-04T14:00:33.658Z 12 240-259 6 5 6 12 9 12 F F F <---
8202024-02-04T14:00:33.658Z 13 260-279 5 5 5 9 9 9 F F F
8212024-02-04T14:00:33.658Z 14 280-299 1 1 1 1 1 1 T F T <---
8222024-02-04T14:00:33.658Z 15 300-319 6 5 6 13 9 13 F F F <---
8232024-02-04T14:00:33.658Z 16 320-339 6 3 6 13 6 13 F F F <---
8242024-02-04T14:00:33.658Z 17 340-359 3 3 3 6 6 6 F F F
8252024-02-04T14:00:33.658Z 18 360-379 2 2 2 2 2 2 F F F
8262024-02-04T14:00:33.658Z 19 380-399 6 4 6 12 8 12 F F F <---
8272024-02-04T14:00:33.658Z 20 400-419 3 3 3 6 6 6 F F F
8282024-02-04T14:00:33.658Z 21 420-439 1 1 1 1 1 1 F F F
8292024-02-04T14:00:33.658Z 22 440-459 6 1 6 13 1 13 F F F <---
8302024-02-04T14:00:33.658Z 23 460-479 6 2 6 13 4 13 F F F <---
8312024-02-04T14:00:33.659Z 24 480-499 1 1 1 1 1 1 F F F
8322024-02-04T14:00:33.659Z 25 500-519 3 3 3 6 6 6 F F F
8332024-02-04T14:00:33.659Z 26 520-539 6 1 6 12 1 12 F F F <---
8342024-02-04T14:00:33.659Z 27 540-559 5 5 5 9 9 9 F F F
8352024-02-04T14:00:33.659Z 28 560-579 5 5 5 9 9 9 F F F
8362024-02-04T14:00:33.659Z 29 580-599 6 1 6 12 1 12 F F F <---
8372024-02-04T14:00:33.659ZMax gen: 6, Max flush: 13
8382024-02-04T14:00:33.659ZError: Difference in extent metadata found!
8392024-02-04T14:00:33.663ZA Difference in extent metadata is expected here
8402024-02-04T14:00:33.663ZOn loop 3
8412024-02-04T14:00:33.663Z
8422024-02-04T14:00:33.666ZVerifying data now
8432024-02-04T14:00:38.973ZNo change after verify
8442024-02-04T14:00:38.973ZLoop: 3 Downstairs dump after verify (and repair):
8452024-02-04T14:00:38.999ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8462024-02-04T14:00:38.999Z 0 000-019 2 2 2 3 3 3 F F F
8472024-02-04T14:00:38.999Z 1 020-039 2 2 2 3 3 3 F F F
8482024-02-04T14:00:38.999Z 2 040-059 3 3 3 6 6 6 F F F
8492024-02-04T14:00:38.999Z 3 060-079 7 7 7 14 14 14 F F F
8502024-02-04T14:00:38.999Z 4 080-099 7 7 7 14 14 14 F F F
8512024-02-04T14:00:38.999Z 5 100-119 6 6 6 13 13 13 F F F
8522024-02-04T14:00:38.999Z 6 120-139 7 7 7 14 14 14 F F F
8532024-02-04T14:00:38.999Z 7 140-159 1 1 1 1 1 1 F F F
8542024-02-04T14:00:38.999Z 8 160-179 6 6 6 13 13 13 F F F
8552024-02-04T14:00:38.999Z 9 180-199 6 6 6 13 13 13 F F F
8562024-02-04T14:00:38.999Z 10 200-219 5 5 5 9 9 9 F F F
8572024-02-04T14:00:39.000Z 11 220-239 1 1 1 1 1 1 F F F
8582024-02-04T14:00:39.005Z 12 240-259 6 6 6 12 12 12 F F F
8592024-02-04T14:00:39.005Z 13 260-279 5 5 5 9 9 9 F F F
8602024-02-04T14:00:39.005Z 14 280-299 7 7 7 14 14 14 F F F
8612024-02-04T14:00:39.005Z 15 300-319 6 6 6 13 13 13 F F F
8622024-02-04T14:00:39.005Z 16 320-339 6 6 6 13 13 13 F F F
8632024-02-04T14:00:39.005Z 17 340-359 3 3 3 6 6 6 F F F
8642024-02-04T14:00:39.006Z 18 360-379 2 2 2 2 2 2 F F F
8652024-02-04T14:00:39.006Z 19 380-399 6 6 6 12 12 12 F F F
8662024-02-04T14:00:39.006Z 20 400-419 3 3 3 6 6 6 F F F
8672024-02-04T14:00:39.006Z 21 420-439 1 1 1 1 1 1 F F F
8682024-02-04T14:00:39.006Z 22 440-459 6 6 6 13 13 13 F F F
8692024-02-04T14:00:39.006Z 23 460-479 6 6 6 13 13 13 F F F
8702024-02-04T14:00:39.006Z 24 480-499 1 1 1 1 1 1 F F F
8712024-02-04T14:00:39.006Z 25 500-519 3 3 3 6 6 6 F F F
8722024-02-04T14:00:39.006Z 26 520-539 6 6 6 12 12 12 F F F
8732024-02-04T14:00:39.006Z 27 540-559 5 5 5 9 9 9 F F F
8742024-02-04T14:00:39.006Z 28 560-579 5 5 5 9 9 9 F F F
8752024-02-04T14:00:39.006Z 29 580-599 6 6 6 12 12 12 F F F
8762024-02-04T14:00:39.006ZMax gen: 7, Max flush: 14
8772024-02-04T14:00:39.006Z
8782024-02-04T14:00:39.006ZBegin loop 4
8792024-02-04T14:00:39.006ZDownstairs to restart: 2
8802024-02-04T14:00:39.039Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.067676833Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161}
8812024-02-04T14:00:39.039Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068164457Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161}
8822024-02-04T14:00:39.040Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068231013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161}
8832024-02-04T14:00:39.046Z{"msg":"Crucible 6b48eda3-6f4e-46f2-ac1f-68ffb05aa69d has session id: fc9f48b6-e27c-4a61-8019-a90ddb557677","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068296083Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8842024-02-04T14:00:39.046Z{"msg":"Upstairs opts: Upstairs UUID: 6b48eda3-6f4e-46f2-ac1f-68ffb05aa69d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
8852024-02-04T14:00:39.046Z,"time":"2024-02-04T14:00:38.06834991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8862024-02-04T14:00:39.046Z{"msg":"Crucible stats registered with UUID: 6b48eda3-6f4e-46f2-ac1f-68ffb05aa69d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068428321Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8872024-02-04T14:00:39.046Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068479047Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161}
8882024-02-04T14:00:39.046Z{"msg":"6b48eda3-6f4e-46f2-ac1f-68ffb05aa69d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068661708Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8892024-02-04T14:00:39.046Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068725662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8902024-02-04T14:00:39.046Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.06877029Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8912024-02-04T14:00:39.046Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068809561Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8922024-02-04T14:00:39.046Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068896284Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8932024-02-04T14:00:39.046Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.068967523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8942024-02-04T14:00:39.046Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.069013682Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8952024-02-04T14:00:39.047Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.069083699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8962024-02-04T14:00:39.047Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.069155424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8972024-02-04T14:00:39.047Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.069203251Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8982024-02-04T14:00:39.047Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.069318318Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
8992024-02-04T14:00:39.047Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.069436282Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9002024-02-04T14:00:39.047Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.069547152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9012024-02-04T14:00:39.047Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.070761181Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9022024-02-04T14:00:39.047Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.070839559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9032024-02-04T14:00:39.047Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.070911268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9042024-02-04T14:00:39.047Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.07095295Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9052024-02-04T14:00:39.047Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.071696413Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9062024-02-04T14:00:39.047Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.071958731Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9072024-02-04T14:00:39.047Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.072025652Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9082024-02-04T14:00:39.047Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.072112161Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9092024-02-04T14:00:39.047Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.072579774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9102024-02-04T14:00:39.047Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.072655669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9112024-02-04T14:00:39.047Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.072684945Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9122024-02-04T14:00:39.047Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.072945496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9132024-02-04T14:00:39.047Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.073001265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9142024-02-04T14:00:39.047Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.07302637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9152024-02-04T14:00:39.047Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.073049719Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9162024-02-04T14:00:39.047Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.073728051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9172024-02-04T14:00:39.047Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.073810693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9182024-02-04T14:00:39.047Z{"msg":"[0]R flush_numbers[0..12]: [3, 3, 6, 14, 14, 13, 14, 1, 13, 13, 9, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.073856401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9192024-02-04T14:00:39.048Z{"msg":"[0]R generation[0..12]: [2, 2, 3, 7, 7, 6, 7, 1, 6, 6, 5, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.073916196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9202024-02-04T14:00:39.048Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.073963183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9212024-02-04T14:00:39.048Z{"msg":"[1]R flush_numbers[0..12]: [3, 3, 6, 14, 14, 13, 14, 1, 13, 13, 9, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074024621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"Wait for a query_work_queue command to finish before sending IO
9222024-02-04T14:00:39.048Z}
9232024-02-04T14:00:39.048Z{"msg":"[1]R generation[0..12]: [2, 2, 3, 7, 7, 6, 7, 1, 6, 6, 5, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074073923Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9242024-02-04T14:00:39.048Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074121076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9252024-02-04T14:00:39.048Z{"msg":"[2]R flush_numbers[0..12]: [3, 3, 6, 14, 14, 13, 14, 1, 13, 13, 9, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074226203Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9262024-02-04T14:00:39.048Z{"msg":"[2]R generation[0..12]: [2, 2, 3, 7, 7, 6, 7, 1, 6, 6, 5, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074272817Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9272024-02-04T14:00:39.048Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074340076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9282024-02-04T14:00:39.048Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.07438465Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9292024-02-04T14:00:39.048Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074451127Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9302024-02-04T14:00:39.048Z{"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074492632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9312024-02-04T14:00:39.048Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074547438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9322024-02-04T14:00:39.048Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.07459027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9332024-02-04T14:00:39.048Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074645762Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9342024-02-04T14:00:39.048Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074688396Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9352024-02-04T14:00:39.048Z{"msg":"Transition from WaitQuorum to ActiveRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9362024-02-04T14:00:39.048Z","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074759339Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9372024-02-04T14:00:39.048Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074806123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9382024-02-04T14:00:39.048Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074863655Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9392024-02-04T14:00:39.048Z{"msg":"6b48eda3-6f4e-46f2-ac1f-68ffb05aa69d is now active with session: fc9f48b6-e27c-4a61-8019-a90ddb557677","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074906577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9402024-02-04T14:00:39.048Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.074959915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161,"session_id":"fc9f48b6-e27c-4a61-8019-a90ddb557677"}
9412024-02-04T14:00:39.048Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:38.075000627Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161}
9422024-02-04T14:00:39.048ZLoading write count information from file "/tmp/test_repair_verify.data"
9432024-02-04T14:00:39.049ZRun Repair workload
9442024-02-04T14:00:39.049Z01/30 Flush
9452024-02-04T14:00:39.049Z02/30 Write block 288 len 20480 data: 2 2 2 2 2
9462024-02-04T14:00:39.052Z03/30 Read block 529 len 8192
9472024-02-04T14:00:39.052Z04/30 Write block 170 len 20480 data: 3 3 4 4 4
9482024-02-04T14:00:39.052Z05/30 Read block 171 len 8192
9492024-02-04T14:00:39.052Z06/30 Read block 277 len 36864
9502024-02-04T14:00:39.052Z07/30 Read block 528 len 16384
9512024-02-04T14:00:39.055Z08/30 Write block 496 len 24576 data: 2 2 2 2 2 2
9522024-02-04T14:00:39.055Z09/30 Read block 546 len 20480
9532024-02-04T14:00:39.055Z10/30 Read block 329 len 24576
9542024-02-04T14:00:39.058Z11/30 Read block 317 len 12288
9552024-02-04T14:00:39.058Z12/30 Write block 216 len 12288 data: 2 2 2
9562024-02-04T14:00:39.058Z13/30 Read block 255 len 8192
9572024-02-04T14:00:39.062Z14/30 Read block 215 len 20480
9582024-02-04T14:00:39.062Z15/30 Read block 297 len 4096
9592024-02-04T14:00:39.065Z16/30 Write block 215 len 12288 data: 2 3 3
9602024-02-04T14:00:39.065Z17/30 Read block 130 len 16384
9612024-02-04T14:00:39.065Z18/30 Read block 428 len 8192
9622024-02-04T14:00:39.067Z19/30 Write block 330 len 32768 data: 2 2 2 2 3 3 3 3
9632024-02-04T14:00:39.067Z20/30 Write block 484 len 28672 data: 2 2 2 2 2 2 2
9642024-02-04T14:00:39.070Z21/30 Read block 466 len 28672
9652024-02-04T14:00:39.070Z22/30 Flush
9662024-02-04T14:00:39.073Z23/30 Write block 488 len 4096 data: 3
9672024-02-04T14:00:39.077Z24/30 Write block 409 len 12288 data: 2 2 3
9682024-02-04T14:00:39.077Z25/30 Write block 581 len 28672 data: 2 2 2 2 2 2 2
9692024-02-04T14:00:39.077Z26/30 Read block 138 len 16384
9702024-02-04T14:00:39.080Z27/30 Write block 302 len 12288 data: 2 2 2
9712024-02-04T14:00:39.080Z28/30 Write block 169 len 32768 data: 3 4 4 5 5 5 4 3
9722024-02-04T14:00:39.080Z29/30 Read block 553 len 12288
9732024-02-04T14:00:39.083Z30/30 Write block 374 len 32768 data: 2 2 2 2 2 3 3 2
9742024-02-04T14:00:39.083Z----------------------------------------------------------------
9752024-02-04T14:00:39.083Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9762024-02-04T14:00:39.083ZStates: Active Active Active
9772024-02-04T14:00:39.084ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9782024-02-04T14:00:39.084Z 1 Acked 1000 Flush 0 Done Done Sent false
9792024-02-04T14:00:39.084Z 2 Acked 1001 Write 5 Done Done Sent false
9802024-02-04T14:00:39.084Z 3 Acked 1002 Read 2 Done Done Sent false
9812024-02-04T14:00:39.084Z 4 Acked 1003 Write 5 Done Done Sent false
9822024-02-04T14:00:39.084Z 5 Acked 1004 Read 2 Done Done Sent false
9832024-02-04T14:00:39.084Z 6 Acked 1005 Read 9 Done Done Sent false
9842024-02-04T14:00:39.084Z 7 Acked 1006 Read 4 Done Done Sent false
9852024-02-04T14:00:39.084Z 8 Acked 1007 Write 6 Done Done Sent false
9862024-02-04T14:00:39.084Z 9 Acked 1008 Read 5 Done Done Sent false
9872024-02-04T14:00:39.084Z 10 Acked 1009 Read 6 Done Done Sent false
9882024-02-04T14:00:39.084Z 11 Acked 1010 Read 3 Done Done Sent false
9892024-02-04T14:00:39.084Z 12 Acked 1011 Write 3 Done Done Sent false
9902024-02-04T14:00:39.084Z 13 Acked 1012 Read 2 Done Done Sent false
9912024-02-04T14:00:39.084Z 14 Acked 1013 Read 5 Done Done Sent false
9922024-02-04T14:00:39.084Z 15 Acked 1014 Read 1 Done Done Sent false
9932024-02-04T14:00:39.084Z 16 Acked 1015 Write 3 Done Done Sent false
9942024-02-04T14:00:39.084Z 17 Acked 1016 Read 4 Done Done Sent false
9952024-02-04T14:00:39.084Z 18 Acked 1017 Read 2 Done Done Sent false
9962024-02-04T14:00:39.084Z 19 Acked 1018 Write 8 Done Done Sent false
9972024-02-04T14:00:39.084Z 20 Acked 1019 Write 7 Done Done Sent false
9982024-02-04T14:00:39.084Z 21 Acked 1020 Read 7 Done Done Sent false
9992024-02-04T14:00:39.084Z 22 Acked 1021 Flush 0 Done Done Sent false
10002024-02-04T14:00:39.084Z 23 Acked 1022 Write 1 Done Done Sent false
10012024-02-04T14:00:39.085Z 24 Acked 1023 Write 3 Done Done Sent false
10022024-02-04T14:00:39.085Z 25 Acked 1024 Write 7 Done Done Sent false
10032024-02-04T14:00:39.085Z 26 Acked 1025 Read 4 Done Done Sent false
10042024-02-04T14:00:39.085Z 27 Acked 1026 Write 3 Done Done Sent false
10052024-02-04T14:00:39.085Z 28 Acked 1027 Write 8 Done Done Sent false
10062024-02-04T14:00:39.085Z 29 Acked 1028 Read 3 Done Done Sent false
10072024-02-04T14:00:39.085Z 30 Acked 1029 Write 8 Sent Sent Sent false
10082024-02-04T14:00:39.085Z STATES DS:0 DS:1 DS:2 TOTAL
10092024-02-04T14:00:39.085Z New 0 0 0 0
10102024-02-04T14:00:39.085Z Sent 1 1 30 32
10112024-02-04T14:00:39.085Z Done 29 29 0 58
10122024-02-04T14:00:39.085Z Skipped 0 0 0 0
10132024-02-04T14:00:39.085Z Error 0 0 0 0
10142024-02-04T14:00:39.085ZLast Flush: 1021 1021 0
10152024-02-04T14:00:39.085ZDownstairs last five completed:
10162024-02-04T14:00:39.085ZUpstairs last five completed: 30 29 28 27 26
10172024-02-04T14:00:39.088Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:00:38.117427275Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1161}
10182024-02-04T14:00:39.088ZWrote out file "/tmp/test_repair_verify.data"
10192024-02-04T14:00:39.091Z
10202024-02-04T14:00:39.097ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10212024-02-04T14:00:39.124ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10222024-02-04T14:00:39.124Z 0 000-019 2 2 2 3 3 3 F F F
10232024-02-04T14:00:39.124Z 1 020-039 2 2 2 3 3 3 F F F
10242024-02-04T14:00:39.125Z 2 040-059 3 3 3 6 6 6 F F F
10252024-02-04T14:00:39.125Z 3 060-079 7 7 7 14 14 14 F F F
10262024-02-04T14:00:39.125Z 4 080-099 7 7 7 14 14 14 F F F
10272024-02-04T14:00:39.125Z 5 100-119 6 6 6 13 13 13 F F F
10282024-02-04T14:00:39.125Z 6 120-139 7 7 7 14 14 14 F F F
10292024-02-04T14:00:39.125Z 7 140-159 1 1 1 1 1 1 F F F
10302024-02-04T14:00:39.125Z 8 160-179 8 8 6 16 16 13 T T F <---
10312024-02-04T14:00:39.125Z 9 180-199 6 6 6 13 13 13 F F F
10322024-02-04T14:00:39.125Z 10 200-219 8 8 5 16 16 9 F F F <---
10332024-02-04T14:00:39.125Z 11 220-239 1 1 1 1 1 1 F F F
10342024-02-04T14:00:39.125Z 12 240-259 6 6 6 12 12 12 F F F
10352024-02-04T14:00:39.125Z 13 260-279 5 5 5 9 9 9 F F F
10362024-02-04T14:00:39.125Z 14 280-299 8 8 7 16 16 14 F F F <---
10372024-02-04T14:00:39.125Z 15 300-319 6 6 6 13 13 13 T T F <---
10382024-02-04T14:00:39.125Z 16 320-339 8 8 6 16 16 13 F F F <---
10392024-02-04T14:00:39.125Z 17 340-359 3 3 3 6 6 6 F F F
10402024-02-04T14:00:39.125Z 18 360-379 2 2 2 2 2 2 T T F <---
10412024-02-04T14:00:39.128Z 19 380-399 6 6 6 12 12 12 T T F <---
10422024-02-04T14:00:39.129Z 20 400-419 3 3 3 6 6 6 T T F <---
10432024-02-04T14:00:39.129Z 21 420-439 1 1 1 1 1 1 F F F
10442024-02-04T14:00:39.129Z 22 440-459 6 6 6 13 13 13 F F F
10452024-02-04T14:00:39.129Z 23 460-479 6 6 6 13 13 13 F F F
10462024-02-04T14:00:39.129Z 24 480-499 8 8 1 16 16 1 T T F <---
10472024-02-04T14:00:39.129Z 25 500-519 8 8 3 16 16 6 F F F <---
10482024-02-04T14:00:39.129Z 26 520-539 6 6 6 12 12 12 F F F
10492024-02-04T14:00:39.129Z 27 540-559 5 5 5 9 9 9 F F F
10502024-02-04T14:00:39.129Z 28 560-579 5 5 5 9 9 9 F F F
10512024-02-04T14:00:39.129Z 29 580-599 6 6 6 12 12 12 T T F <---
10522024-02-04T14:00:39.129ZMax gen: 8, Max flush: 16
10532024-02-04T14:00:39.129ZError: Difference in extent metadata found!
10542024-02-04T14:00:39.132ZA Difference in extent metadata is expected here
10552024-02-04T14:00:39.132ZOn loop 4
10562024-02-04T14:00:39.132Z
10572024-02-04T14:00:39.153ZVerifying data now
10582024-02-04T14:00:44.125ZNo change after verify
10592024-02-04T14:00:44.125ZLoop: 4 Downstairs dump after verify (and repair):
10602024-02-04T14:00:44.146ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10612024-02-04T14:00:44.146Z 0 000-019 2 2 2 3 3 3 F F F
10622024-02-04T14:00:44.146Z 1 020-039 2 2 2 3 3 3 F F F
10632024-02-04T14:00:44.146Z 2 040-059 3 3 3 6 6 6 F F F
10642024-02-04T14:00:44.147Z 3 060-079 7 7 7 14 14 14 F F F
10652024-02-04T14:00:44.147Z 4 080-099 7 7 7 14 14 14 F F F
10662024-02-04T14:00:44.147Z 5 100-119 6 6 6 13 13 13 F F F
10672024-02-04T14:00:44.147Z 6 120-139 7 7 7 14 14 14 F F F
10682024-02-04T14:00:44.147Z 7 140-159 1 1 1 1 1 1 F F F
10692024-02-04T14:00:44.147Z 8 160-179 9 9 9 17 17 17 F F F
10702024-02-04T14:00:44.147Z 9 180-199 6 6 6 13 13 13 F F F
10712024-02-04T14:00:44.147Z 10 200-219 8 8 8 16 16 16 F F F
10722024-02-04T14:00:44.150Z 11 220-239 1 1 1 1 1 1 F F F
10732024-02-04T14:00:44.151Z 12 240-259 6 6 6 12 12 12 F F F
10742024-02-04T14:00:44.151Z 13 260-279 5 5 5 9 9 9 F F F
10752024-02-04T14:00:44.151Z 14 280-299 8 8 8 16 16 16 F F F
10762024-02-04T14:00:44.151Z 15 300-319 9 9 9 17 17 17 F F F
10772024-02-04T14:00:44.151Z 16 320-339 8 8 8 16 16 16 F F F
10782024-02-04T14:00:44.151Z 17 340-359 3 3 3 6 6 6 F F F
10792024-02-04T14:00:44.151Z 18 360-379 9 9 9 17 17 17 F F F
10802024-02-04T14:00:44.151Z 19 380-399 9 9 9 17 17 17 F F F
10812024-02-04T14:00:44.151Z 20 400-419 9 9 9 17 17 17 F F F
10822024-02-04T14:00:44.151Z 21 420-439 1 1 1 1 1 1 F F F
10832024-02-04T14:00:44.151Z 22 440-459 6 6 6 13 13 13 F F F
10842024-02-04T14:00:44.151Z 23 460-479 6 6 6 13 13 13 F F F
10852024-02-04T14:00:44.151Z 24 480-499 9 9 9 17 17 17 F F F
10862024-02-04T14:00:44.151Z 25 500-519 8 8 8 16 16 16 F F F
10872024-02-04T14:00:44.151Z 26 520-539 6 6 6 12 12 12 F F F
10882024-02-04T14:00:44.151Z 27 540-559 5 5 5 9 9 9 F F F
10892024-02-04T14:00:44.151Z 28 560-579 5 5 5 9 9 9 F F F
10902024-02-04T14:00:44.151Z 29 580-599 9 9 9 17 17 17 F F F
10912024-02-04T14:00:44.152ZMax gen: 9, Max flush: 17
10922024-02-04T14:00:44.155Z
10932024-02-04T14:00:44.155ZBegin loop 5
10942024-02-04T14:00:44.155ZDownstairs to restart: 1
10952024-02-04T14:00:44.182Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.215885934Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169}
10962024-02-04T14:00:44.182Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.216373863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169}
10972024-02-04T14:00:44.183Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.216417143Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169}
10982024-02-04T14:00:44.186Z{"msg":"Crucible 6de86303-3f87-4c17-a749-aa5125b0b16f has session id: c2bad596-a8c9-4a91-907c-a7084158ecd9","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.216489867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
10992024-02-04T14:00:44.186Z{"msg":"Upstairs opts: Upstairs UUID: 6de86303-3f87-4c17-a749-aa5125b0b16f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
11002024-02-04T14:00:44.186Z,"time":"2024-02-04T14:00:43.216567646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11012024-02-04T14:00:44.186Z{"msg":"Crucible stats registered with UUID: 6de86303-3f87-4c17-a749-aa5125b0b16f","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.216620574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11022024-02-04T14:00:44.186Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.216688047Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169}
11032024-02-04T14:00:44.186Z{"msg":"6de86303-3f87-4c17-a749-aa5125b0b16f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.216922462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11042024-02-04T14:00:44.186Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.216982726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11052024-02-04T14:00:44.186Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217018152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11062024-02-04T14:00:44.186Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217091963Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11072024-02-04T14:00:44.186Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217172846Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11082024-02-04T14:00:44.186Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217201383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11092024-02-04T14:00:44.186Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217272703Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11102024-02-04T14:00:44.186Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.21731311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11112024-02-04T14:00:44.187Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217391244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11122024-02-04T14:00:44.187Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217439797Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11132024-02-04T14:00:44.187Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217691348Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11142024-02-04T14:00:44.187Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217741637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11152024-02-04T14:00:44.187Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.217787719Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11162024-02-04T14:00:44.187Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.218967356Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11172024-02-04T14:00:44.187Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.219052051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11182024-02-04T14:00:44.187Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.219095803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11192024-02-04T14:00:44.187Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.219166925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11202024-02-04T14:00:44.191Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.220012267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11212024-02-04T14:00:44.191Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.220131388Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11222024-02-04T14:00:44.191Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.220208293Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11232024-02-04T14:00:44.191Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.220255845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11242024-02-04T14:00:44.191Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.220965654Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11252024-02-04T14:00:44.191Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.221024433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11262024-02-04T14:00:44.191Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.221051302Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11272024-02-04T14:00:44.191Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.221270649Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11282024-02-04T14:00:44.191Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.221374104Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11292024-02-04T14:00:44.191Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.221454972Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11302024-02-04T14:00:44.191Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.221519958Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11312024-02-04T14:00:44.191Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222095651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11322024-02-04T14:00:44.191Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222175963Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11332024-02-04T14:00:44.191Z{"msg":"[0]R flush_numbers[0..12]: [3, 3, 6, 14, 14, 13, 14, 1, 17, 13, 16, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222204152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11342024-02-04T14:00:44.191Z{"msg":"[0]R generation[0..12]: [2, 2, 3, 7, 7, 6, 7, 1, 9, 6, 8, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222227685Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11352024-02-04T14:00:44.191Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222270587Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11362024-02-04T14:00:44.191Z{"msg":"[1]R flush_numbers[0..12]: [3, 3, 6, 14, 14, 13, 14, 1, 17, 13, 16, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222295959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11372024-02-04T14:00:44.191Z{"msg":"[1]R generation[0..12]: [2, 2, 3, 7, 7, 6, 7, 1, 9, 6, 8, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222341108Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11382024-02-04T14:00:44.191Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222367021Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11392024-02-04T14:00:44.192Z{"msg":"[2]R flush_numbers[0..12]: [3, 3, 6, 14, 14, 13, 14, 1, 17, 13, 16, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222422532Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11402024-02-04T14:00:44.192Z{"msg":"[2]R generation[0..12]: [2, 2, 3, 7, 7, 6, 7, 1, 9, 6, 8, 1]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
11412024-02-04T14:00:44.192Z30,"time":"2024-02-04T14:00:43.222458961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11422024-02-04T14:00:44.192Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222485615Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11432024-02-04T14:00:44.192Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.22251666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11442024-02-04T14:00:44.192Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222579252Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11452024-02-04T14:00:44.192Z{"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222615183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11462024-02-04T14:00:44.192Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222650855Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11472024-02-04T14:00:44.192Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.22267508Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11482024-02-04T14:00:44.192Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222695649Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11492024-02-04T14:00:44.192Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222727679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11502024-02-04T14:00:44.192Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222785302Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11512024-02-04T14:00:44.192Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222812018Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11522024-02-04T14:00:44.192Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222834537Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11532024-02-04T14:00:44.192Z{"msg":"6de86303-3f87-4c17-a749-aa5125b0b16f is now active with session: c2bad596-a8c9-4a91-907c-a7084158ecd9","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222862621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11542024-02-04T14:00:44.192Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.22293013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169,"session_id":"c2bad596-a8c9-4a91-907c-a7084158ecd9"}
11552024-02-04T14:00:44.192Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:43.222955657Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169}
11562024-02-04T14:00:44.192ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11572024-02-04T14:00:44.192ZLoading write count information from file "/tmp/test_repair_verify.data"
11582024-02-04T14:00:44.192ZRun Repair workload
11592024-02-04T14:00:44.196Z01/30 Write block 491 len 40960 data: 2 2 2 2 2 3 3 3 3 3
11602024-02-04T14:00:44.196Z02/30 Write block 108 len 16384 data: 3 3 3 3
11612024-02-04T14:00:44.196Z03/30 Write block 266 len 24576 data: 5 5 5 5 4 4
11622024-02-04T14:00:44.196Z04/30 Read block 210 len 40960
11632024-02-04T14:00:44.202Z05/30 Write block 227 len 8192 data: 2 2
11642024-02-04T14:00:44.202Z06/30 Read block 539 len 8192
11652024-02-04T14:00:44.202Z07/30 Write block 430 len 36864 data: 2 2 2 2 2 2 2 2 2
11662024-02-04T14:00:44.202Z08/30 Read block 321 len 20480
11672024-02-04T14:00:44.216Z09/30 Write block 514 len 12288 data: 2 2 2
11682024-02-04T14:00:44.216Z10/30 Read block 326 len 32768
11692024-02-04T14:00:44.216Z11/30 Write block 28 len 16384 data: 2 2 2 2
11702024-02-04T14:00:44.216Z12/30 Write block 364 len 28672 data: 2 2 2 2 2 2 2
11712024-02-04T14:00:44.216Z13/30 Read block 255 len 24576
11722024-02-04T14:00:44.216Z14/30 Write block 485 len 24576 data: 3 3 3 4 3 3
11732024-02-04T14:00:44.216Z15/30 Read block 554 len 20480
11742024-02-04T14:00:44.220Z16/30 Flush
11752024-02-04T14:00:44.220Z17/30 Write block 412 len 16384 data: 3 3 3 3
11762024-02-04T14:00:44.220Z18/30 Read block 100 len 40960
11772024-02-04T14:00:44.220Z19/30 Read block 410 len 16384
11782024-02-04T14:00:44.226Z20/30 Write block 586 len 12288 data: 3 3 2
11792024-02-04T14:00:44.226Z21/30 Read block 134 len 40960
11802024-02-04T14:00:44.228Z22/30 Read block 588 len 12288
11812024-02-04T14:00:44.229Z23/30 Read block 6 len 20480
11822024-02-04T14:00:44.229Z24/30 Read block 338 len 24576
11832024-02-04T14:00:44.229Z25/30 Read block 76 len 8192
11842024-02-04T14:00:44.229Z26/30 Read block 268 len 28672
11852024-02-04T14:00:44.231Z27/30 Write block 109 len 36864 data: 4 4 4 3 4 3 3 3 3
11862024-02-04T14:00:44.232Z28/30 Read block 264 len 40960
11872024-02-04T14:00:44.235Z29/30 Write block 506 len 24576 data: 2 2 2 2 2 3
11882024-02-04T14:00:44.235Z30/30 Read block 78 len 40960
11892024-02-04T14:00:44.238Z----------------------------------------------------------------
11902024-02-04T14:00:44.238Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11912024-02-04T14:00:44.238ZStates: Active Active Active
11922024-02-04T14:00:44.238ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11932024-02-04T14:00:44.238Z 1 Acked 1000 Write 10 Done Done Done false
11942024-02-04T14:00:44.238Z 2 Acked 1001 Write 4 Done Done Done false
11952024-02-04T14:00:44.238Z 3 Acked 1002 Write 6 Done Done Done false
11962024-02-04T14:00:44.238Z 4 Acked 1003 Read 10 Done Done Done false
11972024-02-04T14:00:44.239Z 5 Acked 1004 Write 2 Done Done Done false
11982024-02-04T14:00:44.239Z 6 Acked 1005 Read 2 Done Done Done false
11992024-02-04T14:00:44.239Z 7 Acked 1006 Write 9 Done Done Done false
12002024-02-04T14:00:44.239Z 8 Acked 1007 Read 5 Done Done Done false
12012024-02-04T14:00:44.239Z 9 Acked 1008 Write 3 Done Done Done false
12022024-02-04T14:00:44.239Z 10 Acked 1009 Read 8 Done Done Done false
12032024-02-04T14:00:44.239Z 11 Acked 1010 Write 4 Done Done Done false
12042024-02-04T14:00:44.239Z 12 Acked 1011 Write 7 Done Done Done false
12052024-02-04T14:00:44.239Z 13 Acked 1012 Read 6 Done Done Done false
12062024-02-04T14:00:44.239Z 14 Acked 1013 Write 6 Done Done Done false
12072024-02-04T14:00:44.239Z 15 Acked 1014 Read 5 Done Done Done false
12082024-02-04T14:00:44.239Z 16 Acked 1015 Flush 0 Done Sent Done false
12092024-02-04T14:00:44.239Z 17 Acked 1016 Write 4 Done Sent Done false
12102024-02-04T14:00:44.239Z 18 Acked 1017 Read 10 Done Sent Done false
12112024-02-04T14:00:44.239Z 19 Acked 1018 Read 4 Done Sent Done false
12122024-02-04T14:00:44.239Z 20 Acked 1019 Write 3 Done Sent Done false
12132024-02-04T14:00:44.239Z 21 Acked 1020 Read 10 Done Sent Done false
12142024-02-04T14:00:44.239Z 22 Acked 1021 Read 3 Done Sent Done false
12152024-02-04T14:00:44.239Z 23 Acked 1022 Read 5 Done Sent Done false
12162024-02-04T14:00:44.239Z 24 Acked 1023 Read 6 Done Sent Done false
12172024-02-04T14:00:44.239Z 25 Acked 1024 Read 2 Done Sent Done false
12182024-02-04T14:00:44.239Z 26 Acked 1025 Read 7 Done Sent Done false
12192024-02-04T14:00:44.239Z 27 Acked 1026 Write 9 Done Sent Done false
12202024-02-04T14:00:44.239Z 28 Acked 1027 Read 10 Done Sent Done false
12212024-02-04T14:00:44.239Z 29 Acked 1028 Write 6 Done Sent Done false
12222024-02-04T14:00:44.240Z 30 Acked 1029 Read 10 Done Sent Sent false
12232024-02-04T14:00:44.240Z STATES DS:0 DS:1 DS:2 TOTAL
12242024-02-04T14:00:44.240Z New 0 0 0 0
12252024-02-04T14:00:44.240Z Sent 0 15 1 16
12262024-02-04T14:00:44.240Z Done 30 15 29 74
12272024-02-04T14:00:44.240Z Skipped 0 0 0 0
12282024-02-04T14:00:44.240Z Error 0 0 0 0
12292024-02-04T14:00:44.240ZLast Flush: 1015 0 1015
12302024-02-04T14:00:44.240ZDownstairs last five completed:
12312024-02-04T14:00:44.240ZUpstairs last five completed: 30 29 28 27 26
12322024-02-04T14:00:44.240Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:00:43.271061559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1169}
12332024-02-04T14:00:44.240ZWrote out file "/tmp/test_repair_verify.data"
12342024-02-04T14:00:44.244Z
12352024-02-04T14:00:44.251ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12362024-02-04T14:00:44.277ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12372024-02-04T14:00:44.277Z 0 000-019 2 2 2 3 3 3 F F F
12382024-02-04T14:00:44.277Z 1 020-039 10 2 10 18 3 18 F T F <---
12392024-02-04T14:00:44.278Z 2 040-059 3 3 3 6 6 6 F F F
12402024-02-04T14:00:44.278Z 3 060-079 7 7 7 14 14 14 F F F
12412024-02-04T14:00:44.278Z 4 080-099 7 7 7 14 14 14 F F F
12422024-02-04T14:00:44.281Z 5 100-119 10 6 10 18 13 18 T T T <---
12432024-02-04T14:00:44.281Z 6 120-139 7 7 7 14 14 14 F F F
12442024-02-04T14:00:44.281Z 7 140-159 1 1 1 1 1 1 F F F
12452024-02-04T14:00:44.281Z 8 160-179 9 9 9 17 17 17 F F F
12462024-02-04T14:00:44.281Z 9 180-199 6 6 6 13 13 13 F F F
12472024-02-04T14:00:44.281Z 10 200-219 8 8 8 16 16 16 F F F
12482024-02-04T14:00:44.281Z 11 220-239 10 1 10 18 1 18 F T F <---
12492024-02-04T14:00:44.281Z 12 240-259 6 6 6 12 12 12 F F F
12502024-02-04T14:00:44.281Z 13 260-279 10 5 10 18 9 18 F T F <---
12512024-02-04T14:00:44.281Z 14 280-299 8 8 8 16 16 16 F F F
12522024-02-04T14:00:44.281Z 15 300-319 9 9 9 17 17 17 F F F
12532024-02-04T14:00:44.281Z 16 320-339 8 8 8 16 16 16 F F F
12542024-02-04T14:00:44.281Z 17 340-359 3 3 3 6 6 6 F F F
12552024-02-04T14:00:44.281Z 18 360-379 10 9 10 18 17 18 F T F <---
12562024-02-04T14:00:44.281Z 19 380-399 9 9 9 17 17 17 F F F
12572024-02-04T14:00:44.282Z 20 400-419 9 9 9 17 17 17 T F T <---
12582024-02-04T14:00:44.282Z 21 420-439 10 1 10 18 1 18 F T F <---
12592024-02-04T14:00:44.282Z 22 440-459 6 6 6 13 13 13 F F F
12602024-02-04T14:00:44.282Z 23 460-479 6 6 6 13 13 13 F F F
12612024-02-04T14:00:44.282Z 24 480-499 10 9 10 18 17 18 F T F <---
12622024-02-04T14:00:44.282Z 25 500-519 10 8 10 18 16 18 T T T <---
12632024-02-04T14:00:44.282Z 26 520-539 6 6 6 12 12 12 F F F
12642024-02-04T14:00:44.282Z 27 540-559 5 5 5 9 9 9 F F F
12652024-02-04T14:00:44.282Z 28 560-579 5 5 5 9 9 9 F F F
12662024-02-04T14:00:44.282Z 29 580-599 9 9 9 17 17 17 T F T <---
12672024-02-04T14:00:44.282ZMax gen: 10, Max flush: 18
12682024-02-04T14:00:44.282ZError: Difference in extent metadata found!
12692024-02-04T14:00:44.285ZA Difference in extent metadata is expected here
12702024-02-04T14:00:44.285ZOn loop 5
12712024-02-04T14:00:44.285Z
12722024-02-04T14:00:44.288ZVerifying data now
12732024-02-04T14:00:50.200ZNo change after verify
12742024-02-04T14:00:50.200ZLoop: 5 Downstairs dump after verify (and repair):
12752024-02-04T14:00:50.224ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12762024-02-04T14:00:50.224Z 0 000-019 2 2 2 3 3 3 F F F
12772024-02-04T14:00:50.224Z 1 020-039 10 10 10 18 18 18 F F F
12782024-02-04T14:00:50.224Z 2 040-059 3 3 3 6 6 6 F F F
12792024-02-04T14:00:50.224Z 3 060-079 7 7 7 14 14 14 F F F
12802024-02-04T14:00:50.224Z 4 080-099 7 7 7 14 14 14 F F F
12812024-02-04T14:00:50.224Z 5 100-119 11 11 11 19 19 19 F F F
12822024-02-04T14:00:50.224Z 6 120-139 7 7 7 14 14 14 F F F
12832024-02-04T14:00:50.224Z 7 140-159 1 1 1 1 1 1 F F F
12842024-02-04T14:00:50.224Z 8 160-179 9 9 9 17 17 17 F F F
12852024-02-04T14:00:50.224Z 9 180-199 6 6 6 13 13 13 F F F
12862024-02-04T14:00:50.224Z 10 200-219 8 8 8 16 16 16 F F F
12872024-02-04T14:00:50.224Z 11 220-239 10 10 10 18 18 18 F F F
12882024-02-04T14:00:50.224Z 12 240-259 6 6 6 12 12 12 F F F
12892024-02-04T14:00:50.224Z 13 260-279 10 10 10 18 18 18 F F F
12902024-02-04T14:00:50.225Z 14 280-299 8 8 8 16 16 16 F F F
12912024-02-04T14:00:50.231Z 15 300-319 9 9 9 17 17 17 F F F
12922024-02-04T14:00:50.231Z 16 320-339 8 8 8 16 16 16 F F F
12932024-02-04T14:00:50.231Z 17 340-359 3 3 3 6 6 6 F F F
12942024-02-04T14:00:50.231Z 18 360-379 10 10 10 18 18 18 F F F
12952024-02-04T14:00:50.231Z 19 380-399 9 9 9 17 17 17 F F F
12962024-02-04T14:00:50.231Z 20 400-419 11 11 11 19 19 19 F F F
12972024-02-04T14:00:50.231Z 21 420-439 10 10 10 18 18 18 F F F
12982024-02-04T14:00:50.231Z 22 440-459 6 6 6 13 13 13 F F F
12992024-02-04T14:00:50.232Z 23 460-479 6 6 6 13 13 13 F F F
13002024-02-04T14:00:50.232Z 24 480-499 10 10 10 18 18 18 F F F
13012024-02-04T14:00:50.232Z 25 500-519 11 11 11 19 19 19 F F F
13022024-02-04T14:00:50.232Z 26 520-539 6 6 6 12 12 12 F F F
13032024-02-04T14:00:50.232Z 27 540-559 5 5 5 9 9 9 F F F
13042024-02-04T14:00:50.232Z 28 560-579 5 5 5 9 9 9 F F F
13052024-02-04T14:00:50.232Z 29 580-599 11 11 11 19 19 19 F F F
13062024-02-04T14:00:50.232ZMax gen: 11, Max flush: 19
13072024-02-04T14:00:50.232Z
13082024-02-04T14:00:50.232ZBegin loop 6
13092024-02-04T14:00:50.232ZDownstairs to restart: 1
13102024-02-04T14:00:50.260Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.293165074Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177}
13112024-02-04T14:00:50.260Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.293612207Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177}
13122024-02-04T14:00:50.263Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
13132024-02-04T14:00:50.263Z,"time":"2024-02-04T14:00:49.293678231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177}
13142024-02-04T14:00:50.264Z{"msg":"Crucible 48e374b8-53fe-4dc3-a527-5d91eb68c29a has session id: 57525455-aba5-42f2-9f19-f79a25f6248c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.293893743Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13152024-02-04T14:00:50.264Z{"msg":"Upstairs opts: Upstairs UUID: 48e374b8-53fe-4dc3-a527-5d91eb68c29a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.293983328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13162024-02-04T14:00:50.264Z{"msg":"Crucible stats registered with UUID: 48e374b8-53fe-4dc3-a527-5d91eb68c29a","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294061382Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13172024-02-04T14:00:50.264Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294103677Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177}
13182024-02-04T14:00:50.264Z{"msg":"48e374b8-53fe-4dc3-a527-5d91eb68c29a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294167375Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13192024-02-04T14:00:50.264Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294204652Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13202024-02-04T14:00:50.264Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294298964Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13212024-02-04T14:00:50.264Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294371306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13222024-02-04T14:00:50.264Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294414856Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13232024-02-04T14:00:50.264Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294455093Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13242024-02-04T14:00:50.264Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294495432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13252024-02-04T14:00:50.264Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294535175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13262024-02-04T14:00:50.265Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294647705Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13272024-02-04T14:00:50.265Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294719276Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13282024-02-04T14:00:50.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294760358Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13292024-02-04T14:00:50.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294799037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13302024-02-04T14:00:50.265Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.294869634Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13312024-02-04T14:00:50.265Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.296166558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13322024-02-04T14:00:50.265Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.296253241Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13332024-02-04T14:00:50.265Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.296337937Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13342024-02-04T14:00:50.265Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.296406696Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13352024-02-04T14:00:50.270Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.297198343Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13362024-02-04T14:00:50.270Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.297264551Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13372024-02-04T14:00:50.270Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.297577066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13382024-02-04T14:00:50.270Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.297639076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13392024-02-04T14:00:50.270Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.298295843Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13402024-02-04T14:00:50.270Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.298532368Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13412024-02-04T14:00:50.270Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.298601327Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13422024-02-04T14:00:50.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.298704649Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13432024-02-04T14:00:50.271Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.298758572Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13442024-02-04T14:00:50.271Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.298801508Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13452024-02-04T14:00:50.271Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.298856443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13462024-02-04T14:00:50.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299373896Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13472024-02-04T14:00:50.271Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299433624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13482024-02-04T14:00:50.271Z{"msg":"[0]R flush_numbers[0..12]: [3, 18, 6, 14, 14, 19, 14, 1, 17, 13, 16, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299479001Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13492024-02-04T14:00:50.271Z{"msg":"[0]R generation[0..12]: [2, 10, 3, 7, 7, 11, 7, 1, 9, 6, 8, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299533866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13502024-02-04T14:00:50.271Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299579117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13512024-02-04T14:00:50.271Z{"msg":"[1]R flush_numbers[0..12]: [3, 18, 6, 14, 14, 19, 14, 1, 17, 13, 16, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299623536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13522024-02-04T14:00:50.271Z{"msg":"[1]R generation[0..12]: [2, 10, 3, 7, 7, 11, 7, 1, 9, 6, 8, 10]","v":0Wait for a query_work_queue command to finish before sending IO
13532024-02-04T14:00:50.271Z,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299709209Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13542024-02-04T14:00:50.272Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299775861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13552024-02-04T14:00:50.272Z{"msg":"[2]R flush_numbers[0..12]: [3, 18, 6, 14, 14, 19, 14, 1, 17, 13, 16, 18]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299820274Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13562024-02-04T14:00:50.272Z{"msg":"[2]R generation[0..12]: [2, 10, 3, 7, 7, 11, 7, 1, 9, 6, 8, 10]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299885878Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13572024-02-04T14:00:50.272Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299927877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13582024-02-04T14:00:50.272Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.299967129Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13592024-02-04T14:00:50.272Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300003525Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13602024-02-04T14:00:50.272Z{"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300039609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13612024-02-04T14:00:50.272Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.30009937Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13622024-02-04T14:00:50.272Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300151544Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13632024-02-04T14:00:50.272Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300203566Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13642024-02-04T14:00:50.272Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.30024573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13652024-02-04T14:00:50.272Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.30030336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13662024-02-04T14:00:50.272Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300344695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13672024-02-04T14:00:50.273Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300401459Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13682024-02-04T14:00:50.273Zip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13692024-02-04T14:00:50.273Z{"msg":"48e374b8-53fe-4dc3-a527-5d91eb68c29a is now active with session: 57525455-aba5-42f2-9f19-f79a25f6248c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300479005Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13702024-02-04T14:00:50.273Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300540691Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177,"session_id":"57525455-aba5-42f2-9f19-f79a25f6248c"}
13712024-02-04T14:00:50.273Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:49.300575278Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177}
13722024-02-04T14:00:50.273ZLoading write count information from file "/tmp/test_repair_verify.data"
13732024-02-04T14:00:50.273ZRun Repair workload
13742024-02-04T14:00:50.273Z01/30 Write block 419 len 20480 data: 2 2 2 2 2
13752024-02-04T14:00:50.277Z02/30 Write block 452 len 28672 data: 2 2 3 3 3 3 3
13762024-02-04T14:00:50.277Z03/30 Write block 498 len 16384 data: 4 4 4 3
13772024-02-04T14:00:50.277Z04/30 Read block 389 len 32768
13782024-02-04T14:00:50.277Z05/30 Read block 492 len 36864
13792024-02-04T14:00:50.280Z06/30 Write block 515 len 20480 data: 3 3 2 2 2
13802024-02-04T14:00:50.280Z07/30 Read block 388 len 4096
13812024-02-04T14:00:50.281Z08/30 Write block 269 len 16384 data: 6 5 5 2
13822024-02-04T14:00:50.281Z09/30 Read block 481 len 20480
13832024-02-04T14:00:50.284Z10/30 Read block 272 len 36864
13842024-02-04T14:00:50.284Z11/30 Write block 117 len 4096 data: 4
13852024-02-04T14:00:50.284Z12/30 Read block 59 len 16384
13862024-02-04T14:00:50.287Z13/30 Write block 176 len 40960 data: 4 4 4 4 4 4 3 3 3 3
13872024-02-04T14:00:50.287Z14/30 Write block 409 len 8192 data: 3 3
13882024-02-04T14:00:50.294Z15/30 Write block 263 len 4096 data: 5
13892024-02-04T14:00:50.294Z16/30 Read block 289 len 4096
13902024-02-04T14:00:50.294Z17/30 Read block 99 len 4096
13912024-02-04T14:00:50.294Z18/30 Read block 547 len 36864
13922024-02-04T14:00:50.299Z19/30 Write block 469 len 24576 data: 2 2 2 2 2 2
13932024-02-04T14:00:50.299Z20/30 Read block 412 len 8192
13942024-02-04T14:00:50.299Z21/30 Write block 414 len 40960 data: 4 4 2 2 2 3 3 3 3 3
13952024-02-04T14:00:50.299Z22/30 Read block 541 len 32768
13962024-02-04T14:00:50.299Z23/30 Read block 337 len 36864
13972024-02-04T14:00:50.302Z24/30 Write block 255 len 4096 data: 3
13982024-02-04T14:00:50.302Z25/30 Read block 276 len 32768
13992024-02-04T14:00:50.302Z26/30 Flush
14002024-02-04T14:00:50.318Z27/30 Write block 216 len 32768 data: 4 4 3 2 2 2 2 2
14012024-02-04T14:00:50.321Z28/30 Read block 250 len 4096
14022024-02-04T14:00:50.322Z29/30 Write block 192 len 4096 data: 3
14032024-02-04T14:00:50.322Z30/30 Read block 223 len 40960
14042024-02-04T14:00:50.322Z----------------------------------------------------------------
14052024-02-04T14:00:50.322Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14062024-02-04T14:00:50.322ZStates: Active Active Active
14072024-02-04T14:00:50.322ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14082024-02-04T14:00:50.322Z 1 Acked 1000 Write 5 Done Done Done false
14092024-02-04T14:00:50.322Z 2 Acked 1001 Write 7 Done Done Done false
14102024-02-04T14:00:50.322Z 3 Acked 1002 Write 4 Done Sent Done false
14112024-02-04T14:00:50.322Z 4 Acked 1003 Read 8 Done Sent Done false
14122024-02-04T14:00:50.322Z 5 Acked 1004 Read 9 Done Sent Done false
14132024-02-04T14:00:50.322Z 6 Acked 1005 Write 5 Done Sent Done false
14142024-02-04T14:00:50.323Z 7 Acked 1006 Read 1 Done Sent Done false
14152024-02-04T14:00:50.323Z 8 Acked 1007 Write 4 Done Sent Done false
14162024-02-04T14:00:50.323Z 9 Acked 1008 Read 5 Done Sent Done false
14172024-02-04T14:00:50.323Z 10 Acked 1009 Read 9 Done Sent Done false
14182024-02-04T14:00:50.323Z 11 Acked 1010 Write 1 Done Sent Done false
14192024-02-04T14:00:50.323Z 12 Acked 1011 Read 4 Done Sent Done false
14202024-02-04T14:00:50.323Z 13 Acked 1012 Write 10 Done Sent Done false
14212024-02-04T14:00:50.323Z 14 Acked 1013 Write 2 Done Sent Done false
14222024-02-04T14:00:50.323Z 15 Acked 1014 Write 1 Done Sent Done false
14232024-02-04T14:00:50.323Z 16 Acked 1015 Read 1 Done Sent Done false
14242024-02-04T14:00:50.323Z 17 Acked 1016 Read 1 Done Sent Done false
14252024-02-04T14:00:50.323Z 18 Acked 1017 Read 9 Done Sent Done false
14262024-02-04T14:00:50.323Z 19 Acked 1018 Write 6 Done Sent Done false
14272024-02-04T14:00:50.324Z 20 Acked 1019 Read 2 Done Sent Done false
14282024-02-04T14:00:50.324Z 21 Acked 1020 Write 10 Done Sent Done false
14292024-02-04T14:00:50.324Z 22 Acked 1021 Read 8 Done Sent Done false
14302024-02-04T14:00:50.324Z 23 Acked 1022 Read 9 Done Sent Done false
14312024-02-04T14:00:50.324Z 24 Acked 1023 Write 1 Done Sent Done false
14322024-02-04T14:00:50.324Z 25 Acked 1024 Read 8 Done Sent Done false
14332024-02-04T14:00:50.324Z 26 Acked 1025 Flush 0 Done Sent Done false
14342024-02-04T14:00:50.324Z 27 Acked 1026 Write 8 Done Sent Done false
14352024-02-04T14:00:50.324Z 28 Acked 1027 Read 1 Done Sent Done false
14362024-02-04T14:00:50.324Z 29 Acked 1028 Write 1 Done Sent Done false
14372024-02-04T14:00:50.324Z 30 Acked 1029 Read 10 Done Sent Done false
14382024-02-04T14:00:50.324Z STATES DS:0 DS:1 DS:2 TOTAL
14392024-02-04T14:00:50.324Z New 0 0 0 0
14402024-02-04T14:00:50.325Z Sent 0 28 0 28
14412024-02-04T14:00:50.325Z Done 30 2 30 62
14422024-02-04T14:00:50.325Z Skipped 0 0 0 0
14432024-02-04T14:00:50.325Z Error 0 0 0 0
14442024-02-04T14:00:50.325ZLast Flush: 1025 0 1025
14452024-02-04T14:00:50.325ZDownstairs last five completed:
14462024-02-04T14:00:50.325ZUpstairs last five completed: 30 29 28 27 26
14472024-02-04T14:00:50.325Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:00:49.347058946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1177}
14482024-02-04T14:00:50.325ZWrote out file "/tmp/test_repair_verify.data"
14492024-02-04T14:00:50.325Z
14502024-02-04T14:00:50.329ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14512024-02-04T14:00:50.353ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14522024-02-04T14:00:50.353Z 0 000-019 2 2 2 3 3 3 F F F
14532024-02-04T14:00:50.356Z 1 020-039 10 10 10 18 18 18 F F F
14542024-02-04T14:00:50.356Z 2 040-059 3 3 3 6 6 6 F F F
14552024-02-04T14:00:50.357Z 3 060-079 7 7 7 14 14 14 F F F
14562024-02-04T14:00:50.357Z 4 080-099 7 7 7 14 14 14 F F F
14572024-02-04T14:00:50.361Z 5 100-119 12 11 12 20 19 20 F F F <---
14582024-02-04T14:00:50.361Z 6 120-139 7 7 7 14 14 14 F F F
14592024-02-04T14:00:50.361Z 7 140-159 1 1 1 1 1 1 F F F
14602024-02-04T14:00:50.361Z 8 160-179 12 9 12 20 17 20 F F F <---
14612024-02-04T14:00:50.361Z 9 180-199 12 6 12 20 13 20 T F T <---
14622024-02-04T14:00:50.361Z 10 200-219 8 8 8 16 16 16 T F T <---
14632024-02-04T14:00:50.361Z 11 220-239 10 10 10 18 18 18 T F T <---
14642024-02-04T14:00:50.361Z 12 240-259 12 6 12 20 12 20 F F F <---
14652024-02-04T14:00:50.361Z 13 260-279 12 10 12 20 18 20 F F F <---
14662024-02-04T14:00:50.361Z 14 280-299 8 8 8 16 16 16 F F F
14672024-02-04T14:00:50.362Z 15 300-319 9 9 9 17 17 17 F F F
14682024-02-04T14:00:50.362Z 16 320-339 8 8 8 16 16 16 F F F
14692024-02-04T14:00:50.362Z 17 340-359 3 3 3 6 6 6 F F F
14702024-02-04T14:00:50.362Z 18 360-379 10 10 10 18 18 18 F F F
14712024-02-04T14:00:50.362Z 19 380-399 9 9 9 17 17 17 F F F
14722024-02-04T14:00:50.362Z 20 400-419 12 11 12 20 19 20 F T F <---
14732024-02-04T14:00:50.362Z 21 420-439 12 10 12 20 18 20 F T F <---
14742024-02-04T14:00:50.362Z 22 440-459 12 6 12 20 13 20 F T F <---
14752024-02-04T14:00:50.362Z 23 460-479 12 6 12 20 13 20 F F F <---
14762024-02-04T14:00:50.362Z 24 480-499 12 10 12 20 18 20 F F F <---
14772024-02-04T14:00:50.362Z 25 500-519 12 11 12 20 19 20 F F F <---
14782024-02-04T14:00:50.362Z 26 520-539 6 6 6 12 12 12 F F F
14792024-02-04T14:00:50.362Z 27 540-559 5 5 5 9 9 9 F F F
14802024-02-04T14:00:50.362Z 28 560-579 5 5 5 9 9 9 F F F
14812024-02-04T14:00:50.363Z 29 580-599 11 11 11 19 19 19 F F F
14822024-02-04T14:00:50.363ZMax gen: 12, Max flush: 20
14832024-02-04T14:00:50.363ZError: Difference in extent metadata found!
14842024-02-04T14:00:50.363ZA Difference in extent metadata is expected here
14852024-02-04T14:00:50.363ZOn loop 6
14862024-02-04T14:00:50.363Z
14872024-02-04T14:00:50.366ZVerifying data now
14882024-02-04T14:00:56.511ZNo change after verify
14892024-02-04T14:00:56.511ZLoop: 6 Downstairs dump after verify (and repair):
14902024-02-04T14:00:56.534ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14912024-02-04T14:00:56.534Z 0 000-019 2 2 2 3 3 3 F F F
14922024-02-04T14:00:56.534Z 1 020-039 10 10 10 18 18 18 F F F
14932024-02-04T14:00:56.534Z 2 040-059 3 3 3 6 6 6 F F F
14942024-02-04T14:00:56.534Z 3 060-079 7 7 7 14 14 14 F F F
14952024-02-04T14:00:56.534Z 4 080-099 7 7 7 14 14 14 F F F
14962024-02-04T14:00:56.534Z 5 100-119 12 12 12 20 20 20 F F F
14972024-02-04T14:00:56.537Z 6 120-139 7 7 7 14 14 14 F F F
14982024-02-04T14:00:56.537Z 7 140-159 1 1 1 1 1 1 F F F
14992024-02-04T14:00:56.537Z 8 160-179 12 12 12 20 20 20 F F F
15002024-02-04T14:00:56.537Z 9 180-199 13 13 13 21 21 21 F F F
15012024-02-04T14:00:56.537Z 10 200-219 13 13 13 21 21 21 F F F
15022024-02-04T14:00:56.537Z 11 220-239 13 13 13 21 21 21 F F F
15032024-02-04T14:00:56.537Z 12 240-259 12 12 12 20 20 20 F F F
15042024-02-04T14:00:56.537Z 13 260-279 12 12 12 20 20 20 F F F
15052024-02-04T14:00:56.537Z 14 280-299 8 8 8 16 16 16 F F F
15062024-02-04T14:00:56.537Z 15 300-319 9 9 9 17 17 17 F F F
15072024-02-04T14:00:56.537Z 16 320-339 8 8 8 16 16 16 F F F
15082024-02-04T14:00:56.538Z 17 340-359 3 3 3 6 6 6 F F F
15092024-02-04T14:00:56.538Z 18 360-379 10 10 10 18 18 18 F F F
15102024-02-04T14:00:56.538Z 19 380-399 9 9 9 17 17 17 F F F
15112024-02-04T14:00:56.538Z 20 400-419 12 12 12 20 20 20 F F F
15122024-02-04T14:00:56.538Z 21 420-439 12 12 12 20 20 20 F F F
15132024-02-04T14:00:56.538Z 22 440-459 12 12 12 20 20 20 F F F
15142024-02-04T14:00:56.538Z 23 460-479 12 12 12 20 20 20 F F F
15152024-02-04T14:00:56.538Z 24 480-499 12 12 12 20 20 20 F F F
15162024-02-04T14:00:56.538Z 25 500-519 12 12 12 20 20 20 F F F
15172024-02-04T14:00:56.538Z 26 520-539 6 6 6 12 12 12 F F F
15182024-02-04T14:00:56.538Z 27 540-559 5 5 5 9 9 9 F F F
15192024-02-04T14:00:56.538Z 28 560-579 5 5 5 9 9 9 F F F
15202024-02-04T14:00:56.538Z 29 580-599 11 11 11 19 19 19 F F F
15212024-02-04T14:00:56.538ZMax gen: 13, Max flush: 21
15222024-02-04T14:00:56.541Z
15232024-02-04T14:00:56.541ZBegin loop 7
15242024-02-04T14:00:56.541ZDownstairs to restart: 2
15252024-02-04T14:00:56.569Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.602192024Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185}
15262024-02-04T14:00:56.572Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.602732274Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185}
15272024-02-04T14:00:56.572Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
15282024-02-04T14:00:56.572Z2024-02-04T14:00:55.602943905Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185}
15292024-02-04T14:00:56.572Z{"msg":"Crucible 7ff11b55-5a06-4dd4-89ba-a81ca668e938 has session id: 60be359b-5c28-4c3b-9af3-3510a59086f4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603204956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15302024-02-04T14:00:56.572Z{"msg":"Upstairs opts: Upstairs UUID: 7ff11b55-5a06-4dd4-89ba-a81ca668e938, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603292907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15312024-02-04T14:00:56.572Z{"msg":"Crucible stats registered with UUID: 7ff11b55-5a06-4dd4-89ba-a81ca668e938","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603345104Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15322024-02-04T14:00:56.572Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603368462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185}
15332024-02-04T14:00:56.573Z{"msg":"7ff11b55-5a06-4dd4-89ba-a81ca668e938 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.60347428Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15342024-02-04T14:00:56.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603539006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15352024-02-04T14:00:56.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603565285Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15362024-02-04T14:00:56.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.60358958Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15372024-02-04T14:00:56.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603620333Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15382024-02-04T14:00:56.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.60369299Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15392024-02-04T14:00:56.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603752456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15402024-02-04T14:00:56.573Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603781753Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15412024-02-04T14:00:56.573Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603857025Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15422024-02-04T14:00:56.573Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603883625Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15432024-02-04T14:00:56.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.603966989Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15442024-02-04T14:00:56.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.604011611Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15452024-02-04T14:00:56.573Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.604387578Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15462024-02-04T14:00:56.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.605268015Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15472024-02-04T14:00:56.573Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.605334756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15482024-02-04T14:00:56.573Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.605360155Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15492024-02-04T14:00:56.573Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.605383946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15502024-02-04T14:00:56.579Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.606198888Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15512024-02-04T14:00:56.579Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.6063983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15522024-02-04T14:00:56.579Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.606536305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15532024-02-04T14:00:56.579Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.606612828Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15542024-02-04T14:00:56.579Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.607102621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15552024-02-04T14:00:56.579Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.607165577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15562024-02-04T14:00:56.579Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.607246443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15572024-02-04T14:00:56.579Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.60746192Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15582024-02-04T14:00:56.579Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.607514598Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15592024-02-04T14:00:56.579Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.60753984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15602024-02-04T14:00:56.579Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.607597294Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15612024-02-04T14:00:56.579Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608310187Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15622024-02-04T14:00:56.579Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608394731Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15632024-02-04T14:00:56.579Z{"msg":"[0]R flush_numbers[0..12]: [3, 18, 6, 14, 14, 20, 14, 1, 20, 21, 21, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608420863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15642024-02-04T14:00:56.580Z{"msg":"[0]R generation[0..12]: [2, 10, 3, 7, 7, 12, 7, 1, 12, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608463387Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15652024-02-04T14:00:56.580Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608489717Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15662024-02-04T14:00:56.580Z{"msg":"[1]R flush_numbers[0..12]: [3, 18, 6, 14, 14, 20, 14, 1, 20, 21, 21, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608532689Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15672024-02-04T14:00:56.580Z{"msg":"[1]R generation[0..12]: [2, 10, 3, 7, 7, 12, 7, 1, 12, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608557655Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15682024-02-04T14:00:56.580Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
15692024-02-04T14:00:56.580Z,"time":"2024-02-04T14:00:55.608602158Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15702024-02-04T14:00:56.580Z{"msg":"[2]R flush_numbers[0..12]: [3, 18, 6, 14, 14, 20, 14, 1, 20, 21, 21, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608627887Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15712024-02-04T14:00:56.580Z{"msg":"[2]R generation[0..12]: [2, 10, 3, 7, 7, 12, 7, 1, 12, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.60865124Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15722024-02-04T14:00:56.580Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608675348Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15732024-02-04T14:00:56.580Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608697327Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15742024-02-04T14:00:56.580Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608719841Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15752024-02-04T14:00:56.580Z{"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608741624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15762024-02-04T14:00:56.580Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608788876Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15772024-02-04T14:00:56.580Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608812725Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15782024-02-04T14:00:56.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608833346Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15792024-02-04T14:00:56.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608875893Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15802024-02-04T14:00:56.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608899567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15812024-02-04T14:00:56.580Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608922355Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15822024-02-04T14:00:56.580Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.608971943Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15832024-02-04T14:00:56.580Z{"msg":"7ff11b55-5a06-4dd4-89ba-a81ca668e938 is now active with session: 60be359b-5c28-4c3b-9af3-3510a59086f4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.60899516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15842024-02-04T14:00:56.580Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.609015983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185,"session_id":"60be359b-5c28-4c3b-9af3-3510a59086f4"}
15852024-02-04T14:00:56.580Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:00:55.609063262Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185}
15862024-02-04T14:00:56.580ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15872024-02-04T14:00:56.581ZLoading write count information from file "/tmp/test_repair_verify.data"
15882024-02-04T14:00:56.581ZRun Repair workload
15892024-02-04T14:00:56.584Z01/30 Write block 119 len 32768 data: 2 2 2 2 2 2 3 3
15902024-02-04T14:00:56.584Z02/30 Write block 502 len 28672 data: 2 2 2 2 3 3 3
15912024-02-04T14:00:56.584Z03/30 Flush
15922024-02-04T14:00:56.587Z04/30 Write block 449 len 36864 data: 3 2 2 3 3 4 4 4 4
15932024-02-04T14:00:56.587Z05/30 Read block 528 len 28672
15942024-02-04T14:00:56.590Z06/30 Read block 357 len 8192
15952024-02-04T14:00:56.590Z07/30 Read block 9 len 8192
15962024-02-04T14:00:56.593Z08/30 Read block 318 len 16384
15972024-02-04T14:00:56.593Z09/30 Read block 240 len 20480
15982024-02-04T14:00:56.596Z10/30 Write block 509 len 8192 data: 3 3
15992024-02-04T14:00:56.596Z11/30 Write block 309 len 12288 data: 4 4 3
16002024-02-04T14:00:56.596Z12/30 Write block 57 len 32768 data: 2 2 3 4 4 4 3 3
16012024-02-04T14:00:56.599Z13/30 Read block 179 len 16384
16022024-02-04T14:00:56.599Z14/30 Write block 549 len 40960 data: 2 2 2 2 2 2 2 4 4 4
16032024-02-04T14:00:56.601Z15/30 Read block 151 len 12288
16042024-02-04T14:00:56.601Z16/30 Read block 362 len 4096
16052024-02-04T14:00:56.604Z17/30 Write block 378 len 12288 data: 3 4 4
16062024-02-04T14:00:56.604Z18/30 Write block 454 len 4096 data: 5
16072024-02-04T14:00:56.604Z19/30 Read block 488 len 36864
16082024-02-04T14:00:56.607Z20/30 Read block 313 len 16384
16092024-02-04T14:00:56.607Z21/30 Read block 193 len 16384
16102024-02-04T14:00:56.610Z22/30 Read block 83 len 20480
16112024-02-04T14:00:56.610Z23/30 Write block 509 len 16384 data: 4 4 4 2
16122024-02-04T14:00:56.613Z24/30 Read block 432 len 28672
16132024-02-04T14:00:56.613Z25/30 Read block 264 len 16384
16142024-02-04T14:00:56.615Z26/30 Write block 325 len 24576 data: 2 2 2 2 2 3
16152024-02-04T14:00:56.615Z27/30 Write block 433 len 36864 data: 3 3 3 3 3 3 2 2 2
16162024-02-04T14:00:56.621Z28/30 Read block 530 len 40960
16172024-02-04T14:00:56.625Z29/30 Read block 303 len 8192
16182024-02-04T14:00:56.625Z30/30 Write block 541 len 20480 data: 2 2 2 2 2
16192024-02-04T14:00:56.626Z----------------------------------------------------------------
16202024-02-04T14:00:56.626Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:27
16212024-02-04T14:00:56.626ZStates: Active Active Active
16222024-02-04T14:00:56.626ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16232024-02-04T14:00:56.626Z 4 Acked 1003 Write 9 Done Done Done false
16242024-02-04T14:00:56.626Z 5 Acked 1004 Read 7 Done Done Done false
16252024-02-04T14:00:56.626Z 6 Acked 1005 Read 2 Done Done Sent false
16262024-02-04T14:00:56.626Z 7 Acked 1006 Read 2 Done Done Sent false
16272024-02-04T14:00:56.626Z 8 Acked 1007 Read 4 Done Done Sent false
16282024-02-04T14:00:56.626Z 9 Acked 1008 Read 5 Done Done Sent false
16292024-02-04T14:00:56.626Z 10 Acked 1009 Write 2 Done Done Sent false
16302024-02-04T14:00:56.626Z 11 Acked 1010 Write 3 Done Done Sent false
16312024-02-04T14:00:56.626Z 12 Acked 1011 Write 8 Done Done Sent false
16322024-02-04T14:00:56.626Z 13 Acked 1012 Read 4 Done Done Sent false
16332024-02-04T14:00:56.626Z 14 Acked 1013 Write 10 Done Done Sent false
16342024-02-04T14:00:56.626Z 15 Acked 1014 Read 3 Done Done Sent false
16352024-02-04T14:00:56.626Z 16 Acked 1015 Read 1 Done Done Sent false
16362024-02-04T14:00:56.626Z 17 Acked 1016 Write 3 Done Done Sent false
16372024-02-04T14:00:56.626Z 18 Acked 1017 Write 1 Done Done Sent false
16382024-02-04T14:00:56.626Z 19 Acked 1018 Read 9 Done Done Sent false
16392024-02-04T14:00:56.626Z 20 Acked 1019 Read 4 Done Done Sent false
16402024-02-04T14:00:56.626Z 21 Acked 1020 Read 4 Done Done Sent false
16412024-02-04T14:00:56.626Z 22 Acked 1021 Read 5 Done Done Sent false
16422024-02-04T14:00:56.626Z 23 Acked 1022 Write 4 Done Done Sent false
16432024-02-04T14:00:56.627Z 24 Acked 1023 Read 7 Done Done Sent false
16442024-02-04T14:00:56.627Z 25 Acked 1024 Read 4 Done Done Sent false
16452024-02-04T14:00:56.627Z 26 Acked 1025 Write 6 Done Done Sent false
16462024-02-04T14:00:56.627Z 27 Acked 1026 Write 9 Done Done Sent false
16472024-02-04T14:00:56.627Z 28 Acked 1027 Read 10 Done Done Sent false
16482024-02-04T14:00:56.627Z 29 Acked 1028 Read 2 Done Done Sent false
16492024-02-04T14:00:56.627Z 30 Acked 1029 Write 5 Sent Sent Sent false
16502024-02-04T14:00:56.627Z STATES DS:0 DS:1 DS:2 TOTAL
16512024-02-04T14:00:56.627Z New 0 0 0 0
16522024-02-04T14:00:56.627Z Sent 1 1 25 27
16532024-02-04T14:00:56.627Z Done 26 26 2 54
16542024-02-04T14:00:56.627Z Skipped 0 0 0 0
16552024-02-04T14:00:56.627Z Error 0 0 0 0
16562024-02-04T14:00:56.627ZLast Flush: 1002 1002 1002
16572024-02-04T14:00:56.627ZDownstairs last five completed: 1002 1001 1000
16582024-02-04T14:00:56.627ZUpstairs last five completed: 30 29 28 27 26
16592024-02-04T14:00:56.627Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:00:55.654699935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1185}
16602024-02-04T14:00:56.627ZWrote out file "/tmp/test_repair_verify.data"
16612024-02-04T14:00:56.634Z
16622024-02-04T14:00:56.636ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16632024-02-04T14:00:56.661ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16642024-02-04T14:00:56.661Z 0 000-019 2 2 2 3 3 3 F F F
16652024-02-04T14:00:56.661Z 1 020-039 10 10 10 18 18 18 F F F
16662024-02-04T14:00:56.661Z 2 040-059 3 3 3 6 6 6 T T F <---
16672024-02-04T14:00:56.661Z 3 060-079 7 7 7 14 14 14 T T F <---
16682024-02-04T14:00:56.661Z 4 080-099 7 7 7 14 14 14 F F F
16692024-02-04T14:00:56.661Z 5 100-119 14 14 14 22 22 22 F F F
16702024-02-04T14:00:56.661Z 6 120-139 14 14 14 22 22 22 F F F
16712024-02-04T14:00:56.664Z 7 140-159 1 1 1 1 1 1 F F F
16722024-02-04T14:00:56.664Z 8 160-179 12 12 12 20 20 20 F F F
16732024-02-04T14:00:56.665Z 9 180-199 13 13 13 21 21 21 F F F
16742024-02-04T14:00:56.665Z 10 200-219 13 13 13 21 21 21 F F F
16752024-02-04T14:00:56.665Z 11 220-239 13 13 13 21 21 21 F F F
16762024-02-04T14:00:56.665Z 12 240-259 12 12 12 20 20 20 F F F
16772024-02-04T14:00:56.665Z 13 260-279 12 12 12 20 20 20 F F F
16782024-02-04T14:00:56.665Z 14 280-299 8 8 8 16 16 16 F F F
16792024-02-04T14:00:56.665Z 15 300-319 9 9 9 17 17 17 T T F <---
16802024-02-04T14:00:56.665Z 16 320-339 8 8 8 16 16 16 T T F <---
16812024-02-04T14:00:56.665Z 17 340-359 3 3 3 6 6 6 F F F
16822024-02-04T14:00:56.665Z 18 360-379 10 10 10 18 18 18 T T F <---
16832024-02-04T14:00:56.665Z 19 380-399 9 9 9 17 17 17 T T F <---
16842024-02-04T14:00:56.665Z 20 400-419 12 12 12 20 20 20 F F F
16852024-02-04T14:00:56.665Z 21 420-439 12 12 12 20 20 20 T T F <---
16862024-02-04T14:00:56.665Z 22 440-459 12 12 12 20 20 20 T T T
16872024-02-04T14:00:56.665Z 23 460-479 12 12 12 20 20 20 F F F
16882024-02-04T14:00:56.665Z 24 480-499 12 12 12 20 20 20 F F F
16892024-02-04T14:00:56.665Z 25 500-519 14 14 14 22 22 22 T T F <---
16902024-02-04T14:00:56.665Z 26 520-539 6 6 6 12 12 12 F F F
16912024-02-04T14:00:56.665Z 27 540-559 5 5 5 9 9 9 T T F <---
16922024-02-04T14:00:56.665Z 28 560-579 5 5 5 9 9 9 F F F
16932024-02-04T14:00:56.665Z 29 580-599 11 11 11 19 19 19 F F F
16942024-02-04T14:00:56.665ZMax gen: 14, Max flush: 22
16952024-02-04T14:00:56.665ZError: Difference in extent metadata found!
16962024-02-04T14:00:56.668ZA Difference in extent metadata is expected here
16972024-02-04T14:00:56.668ZOn loop 7
16982024-02-04T14:00:56.669Z
16992024-02-04T14:00:56.671ZVerifying data now
17002024-02-04T14:01:01.595ZNo change after verify
17012024-02-04T14:01:01.595ZLoop: 7 Downstairs dump after verify (and repair):
17022024-02-04T14:01:01.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17032024-02-04T14:01:01.617Z 0 000-019 2 2 2 3 3 3 F F F
17042024-02-04T14:01:01.617Z 1 020-039 10 10 10 18 18 18 F F F
17052024-02-04T14:01:01.617Z 2 040-059 15 15 15 23 23 23 F F F
17062024-02-04T14:01:01.617Z 3 060-079 15 15 15 23 23 23 F F F
17072024-02-04T14:01:01.617Z 4 080-099 7 7 7 14 14 14 F F F
17082024-02-04T14:01:01.617Z 5 100-119 14 14 14 22 22 22 F F F
17092024-02-04T14:01:01.617Z 6 120-139 14 14 14 22 22 22 F F F
17102024-02-04T14:01:01.618Z 7 140-159 1 1 1 1 1 1 F F F
17112024-02-04T14:01:01.621Z 8 160-179 12 12 12 20 20 20 F F F
17122024-02-04T14:01:01.621Z 9 180-199 13 13 13 21 21 21 F F F
17132024-02-04T14:01:01.622Z 10 200-219 13 13 13 21 21 21 F F F
17142024-02-04T14:01:01.622Z 11 220-239 13 13 13 21 21 21 F F F
17152024-02-04T14:01:01.622Z 12 240-259 12 12 12 20 20 20 F F F
17162024-02-04T14:01:01.622Z 13 260-279 12 12 12 20 20 20 F F F
17172024-02-04T14:01:01.622Z 14 280-299 8 8 8 16 16 16 F F F
17182024-02-04T14:01:01.622Z 15 300-319 15 15 15 23 23 23 F F F
17192024-02-04T14:01:01.622Z 16 320-339 15 15 15 23 23 23 F F F
17202024-02-04T14:01:01.622Z 17 340-359 3 3 3 6 6 6 F F F
17212024-02-04T14:01:01.622Z 18 360-379 15 15 15 23 23 23 F F F
17222024-02-04T14:01:01.622Z 19 380-399 15 15 15 23 23 23 F F F
17232024-02-04T14:01:01.622Z 20 400-419 12 12 12 20 20 20 F F F
17242024-02-04T14:01:01.622Z 21 420-439 15 15 15 23 23 23 F F F
17252024-02-04T14:01:01.622Z 22 440-459 15 15 15 23 23 23 F F F
17262024-02-04T14:01:01.623Z 23 460-479 12 12 12 20 20 20 F F F
17272024-02-04T14:01:01.623Z 24 480-499 12 12 12 20 20 20 F F F
17282024-02-04T14:01:01.623Z 25 500-519 15 15 15 23 23 23 F F F
17292024-02-04T14:01:01.623Z 26 520-539 6 6 6 12 12 12 F F F
17302024-02-04T14:01:01.623Z 27 540-559 15 15 15 23 23 23 F F F
17312024-02-04T14:01:01.623Z 28 560-579 5 5 5 9 9 9 F F F
17322024-02-04T14:01:01.623Z 29 580-599 11 11 11 19 19 19 F F F
17332024-02-04T14:01:01.623ZMax gen: 15, Max flush: 23
17342024-02-04T14:01:01.627Z
17352024-02-04T14:01:01.627ZBegin loop 8
17362024-02-04T14:01:01.627ZDownstairs to restart: 0
17372024-02-04T14:01:01.654Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.687127137Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193}
17382024-02-04T14:01:01.654Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.687589714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193}
17392024-02-04T14:01:01.654Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.687619934Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193}
17402024-02-04T14:01:01.655Z{"msg":"Crucible 2e4c514e-2854-4338-9624-85844485edfd has session id: b0543a09-fc3b-4df4-bb3f-3a43763fb9e3","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.687655885Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17412024-02-04T14:01:01.659ZCrucible runtime is spawned
17422024-02-04T14:01:01.659Z{"msg":"Upstairs opts: Upstairs UUID: 2e4c514e-2854-4338-9624-85844485edfd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.687894512Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17432024-02-04T14:01:01.659Z{"msg":"Crucible stats registered with UUID: 2e4c514e-2854-4338-9624-85844485edfd","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688018861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17442024-02-04T14:01:01.659Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688158449Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193}
17452024-02-04T14:01:01.659Z{"msg":"2e4c514e-2854-4338-9624-85844485edfd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688181297Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17462024-02-04T14:01:01.659Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688202497Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17472024-02-04T14:01:01.659Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688245057Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17482024-02-04T14:01:01.659Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688269139Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17492024-02-04T14:01:01.660Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.68834022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17502024-02-04T14:01:01.660Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688394464Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17512024-02-04T14:01:01.660Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688606456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17522024-02-04T14:01:01.660Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688658902Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17532024-02-04T14:01:01.660Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688704201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17542024-02-04T14:01:01.660Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688746163Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17552024-02-04T14:01:01.660Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688817963Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17562024-02-04T14:01:01.660Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688891695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17572024-02-04T14:01:01.660Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.688935474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17582024-02-04T14:01:01.660Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.689817773Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17592024-02-04T14:01:01.660Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.689888094Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17602024-02-04T14:01:01.660Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.689914092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17612024-02-04T14:01:01.661Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.689938252Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17622024-02-04T14:01:01.661Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.690737172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17632024-02-04T14:01:01.661Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.691010363Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17642024-02-04T14:01:01.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.691628061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17652024-02-04T14:01:01.661Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.691701641Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17662024-02-04T14:01:01.661Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.691824762Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17672024-02-04T14:01:01.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.691901659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17682024-02-04T14:01:01.661Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.691928167Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17692024-02-04T14:01:01.661Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.691951275Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17702024-02-04T14:01:01.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.692072078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17712024-02-04T14:01:01.674Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.692125006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17722024-02-04T14:01:01.674Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.692886801Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17732024-02-04T14:01:01.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693561828Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17742024-02-04T14:01:01.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693629504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17752024-02-04T14:01:01.674Z{"msg":"[0]R flush_numbers[0..12]: [3, 18, 23, 23, 14, 22, 22, 1, 20, 21, 21, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693657849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17762024-02-04T14:01:01.674Z{"msg":"[0]R generation[0..12]: [2, 10, 15, 15, 7, 14, 14, 1, 12, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693682939Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17772024-02-04T14:01:01.674Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693727616Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17782024-02-04T14:01:01.674Z{"msg":"[1]R flush_numbers[0..12]: [3, 18, 23, 23, 14, 22, 22, 1, 20, 21, 21, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693753376Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17792024-02-04T14:01:01.674Z{"msg":"[1]R generation[0..12]: [2, 10, 15, 15, 7, 14, 14, 1, 12, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693795341Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17802024-02-04T14:01:01.674Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693821377Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17812024-02-04T14:01:01.674Z{"msg":"[2]R flush_numbers[0..12]: [3, 18, 23, 23, 14, 22, 22, 1, 20, 21, 21, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693867643Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17822024-02-04T14:01:01.675Z{"msg":"[2]R generation[0..12]: [2, 10, 15, 15, 7, 14, 14, 1, 12, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693892328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17832024-02-04T14:01:01.675ZWait for a query_work_queue command to finish before sending IO
17842024-02-04T14:01:01.675Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.693920362Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17852024-02-04T14:01:01.675Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.69394291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17862024-02-04T14:01:01.675Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.69396553Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17872024-02-04T14:01:01.675Z{"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.69400729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17882024-02-04T14:01:01.675Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.694030436Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17892024-02-04T14:01:01.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.694052908Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17902024-02-04T14:01:01.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.694099346Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17912024-02-04T14:01:01.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.694123381Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17922024-02-04T14:01:01.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.69414551Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17932024-02-04T14:01:01.675Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.694191589Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17942024-02-04T14:01:01.675Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.694214034Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17952024-02-04T14:01:01.675Z{"msg":"2e4c514e-2854-4338-9624-85844485edfd is now active with session: b0543a09-fc3b-4df4-bb3f-3a43763fb9e3","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.69423625Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17962024-02-04T14:01:01.675Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.694257897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193,"session_id":"b0543a09-fc3b-4df4-bb3f-3a43763fb9e3"}
17972024-02-04T14:01:01.675Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:00.694278993Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193}
17982024-02-04T14:01:01.675ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17992024-02-04T14:01:01.675ZLoading write count information from file "/tmp/test_repair_verify.data"
18002024-02-04T14:01:01.675ZRun Repair workload
18012024-02-04T14:01:01.675Z01/30 Write block 353 len 20480 data: 2 2 2 2 2
18022024-02-04T14:01:01.675Z02/30 Read block 276 len 4096
18032024-02-04T14:01:01.679Z03/30 Read block 576 len 12288
18042024-02-04T14:01:01.679Z04/30 Read block 547 len 24576
18052024-02-04T14:01:01.680Z05/30 Read block 9 len 12288
18062024-02-04T14:01:01.680Z06/30 Write block 174 len 4096 data: 6
18072024-02-04T14:01:01.680Z07/30 Read block 274 len 12288
18082024-02-04T14:01:01.680Z08/30 Write block 273 len 40960 data: 2 2 2 2 2 2 2 2 2 2
18092024-02-04T14:01:01.680Z09/30 Write block 561 len 24576 data: 5 4 4 4 3 3
18102024-02-04T14:01:01.680Z10/30 Write block 400 len 36864 data: 3 2 2 2 2 2 2 2 2
18112024-02-04T14:01:01.680Z11/30 Read block 307 len 32768
18122024-02-04T14:01:01.680Z12/30 Flush
18132024-02-04T14:01:01.683Z13/30 Write block 500 len 32768 data: 5 4 3 3 3 3 4 4
18142024-02-04T14:01:01.687Z14/30 Write block 39 len 4096 data: 2
18152024-02-04T14:01:01.687Z15/30 Flush
18162024-02-04T14:01:01.693Z16/30 Write block 30 len 32768 data: 3 3 2 2 2 2 2 2
18172024-02-04T14:01:01.693Z17/30 Flush
18182024-02-04T14:01:01.693Z18/30 Write block 301 len 16384 data: 2 3 3 3
18192024-02-04T14:01:01.696Z19/30 Flush
18202024-02-04T14:01:01.697Z20/30 Flush
18212024-02-04T14:01:01.700Z21/30 Flush
18222024-02-04T14:01:01.700Z22/30 Flush
18232024-02-04T14:01:01.701Z23/30 Flush
18242024-02-04T14:01:01.701Z24/30 Flush
18252024-02-04T14:01:01.701Z25/30 Write block 304 len 8192 data: 4 3
18262024-02-04T14:01:01.705Z26/30 Write block 198 len 16384 data: 2 3 3 3
18272024-02-04T14:01:01.705Z27/30 Read block 333 len 32768
18282024-02-04T14:01:01.706Z28/30 Read block 158 len 8192
18292024-02-04T14:01:01.709Z29/30 Read block 388 len 40960
18302024-02-04T14:01:01.710Z30/30 Read block 199 len 32768
18312024-02-04T14:01:01.710Z----------------------------------------------------------------
18322024-02-04T14:01:01.710Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18332024-02-04T14:01:01.710ZStates: Active Active Active
18342024-02-04T14:01:01.710ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18352024-02-04T14:01:01.710Z 1 Acked 1000 Write 5 Done Done Done false
18362024-02-04T14:01:01.710Z 2 Acked 1001 Read 1 Done Done Done false
18372024-02-04T14:01:01.710Z 3 Acked 1002 Read 3 Done Done Done false
18382024-02-04T14:01:01.710Z 4 Acked 1003 Read 6 Done Done Done false
18392024-02-04T14:01:01.710Z 5 Acked 1004 Read 3 Sent Done Done false
18402024-02-04T14:01:01.710Z 6 Acked 1005 Write 1 Sent Done Done false
18412024-02-04T14:01:01.710Z 7 Acked 1006 Read 3 Sent Done Done false
18422024-02-04T14:01:01.710Z 8 Acked 1007 Write 10 Sent Done Done false
18432024-02-04T14:01:01.711Z 9 Acked 1008 Write 6 Sent Done Done false
18442024-02-04T14:01:01.711Z 10 Acked 1009 Write 9 Sent Done Done false
18452024-02-04T14:01:01.711Z 11 Acked 1010 Read 8 Sent Done Done false
18462024-02-04T14:01:01.711Z 12 Acked 1011 Flush 0 Sent Done Done false
18472024-02-04T14:01:01.711Z 13 Acked 1012 Write 8 Sent Done Done false
18482024-02-04T14:01:01.711Z 14 Acked 1013 Write 1 Sent Done Done false
18492024-02-04T14:01:01.711Z 15 Acked 1014 Flush 0 Sent Done Done false
18502024-02-04T14:01:01.711Z 16 Acked 1015 Write 8 Sent Done Done false
18512024-02-04T14:01:01.711Z 17 Acked 1016 Flush 0 Sent Done Done false
18522024-02-04T14:01:01.711Z 18 Acked 1017 Write 4 Sent Done Done false
18532024-02-04T14:01:01.711Z 19 Acked 1018 Flush 0 Sent Done Done false
18542024-02-04T14:01:01.711Z 20 Acked 1019 Flush 0 Sent Done Done false
18552024-02-04T14:01:01.711Z 21 Acked 1020 Flush 0 Sent Done Done false
18562024-02-04T14:01:01.712Z 22 Acked 1021 Flush 0 Sent Done Done false
18572024-02-04T14:01:01.712Z 23 Acked 1022 Flush 0 Sent Done Done false
18582024-02-04T14:01:01.712Z 24 Acked 1023 Flush 0 Sent Done Done false
18592024-02-04T14:01:01.712Z 25 Acked 1024 Write 2 Sent Done Done false
18602024-02-04T14:01:01.712Z 26 Acked 1025 Write 4 Sent Done Done false
18612024-02-04T14:01:01.712Z 27 Acked 1026 Read 8 Sent Done Done false
18622024-02-04T14:01:01.712Z 28 Acked 1027 Read 2 Sent Done Done false
18632024-02-04T14:01:01.712Z 29 Acked 1028 Read 10 Sent Done Done false
18642024-02-04T14:01:01.712Z 30 Acked 1029 Read 8 Sent Sent Done false
18652024-02-04T14:01:01.712Z STATES DS:0 DS:1 DS:2 TOTAL
18662024-02-04T14:01:01.712Z New 0 0 0 0
18672024-02-04T14:01:01.712Z Sent 26 1 0 27
18682024-02-04T14:01:01.712Z Done 4 29 30 63
18692024-02-04T14:01:01.713Z Skipped 0 0 0 0
18702024-02-04T14:01:01.713Z Error 0 0 0 0
18712024-02-04T14:01:01.713ZLast Flush: 0 1023 1023
18722024-02-04T14:01:01.713ZDownstairs last five completed:
18732024-02-04T14:01:01.713ZUpstairs last five completed: 30 29 28 27 26
18742024-02-04T14:01:01.713Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:00.742575435Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1193}
18752024-02-04T14:01:01.718ZWrote out file "/tmp/test_repair_verify.data"
18762024-02-04T14:01:01.718Z
18772024-02-04T14:01:01.723ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18782024-02-04T14:01:01.751ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18792024-02-04T14:01:01.751Z 0 000-019 2 2 2 3 3 3 F F F
18802024-02-04T14:01:01.751Z 1 020-039 10 16 16 18 26 26 F F F <---
18812024-02-04T14:01:01.751Z 2 040-059 15 15 15 23 23 23 F F F
18822024-02-04T14:01:01.751Z 3 060-079 15 15 15 23 23 23 F F F
18832024-02-04T14:01:01.751Z 4 080-099 7 7 7 14 14 14 F F F
18842024-02-04T14:01:01.751Z 5 100-119 14 14 14 22 22 22 F F F
18852024-02-04T14:01:01.751Z 6 120-139 14 14 14 22 22 22 F F F
18862024-02-04T14:01:01.752Z 7 140-159 1 1 1 1 1 1 F F F
18872024-02-04T14:01:01.752Z 8 160-179 12 16 16 20 24 24 F F F <---
18882024-02-04T14:01:01.752Z 9 180-199 13 13 13 21 21 21 F T T <---
18892024-02-04T14:01:01.752Z 10 200-219 13 13 13 21 21 21 F T T <---
18902024-02-04T14:01:01.752Z 11 220-239 13 13 13 21 21 21 F F F
18912024-02-04T14:01:01.752Z 12 240-259 12 12 12 20 20 20 F F F
18922024-02-04T14:01:01.752Z 13 260-279 12 16 16 20 24 24 F F F <---
18932024-02-04T14:01:01.756Z 14 280-299 8 16 16 16 24 24 F F F <---
18942024-02-04T14:01:01.756Z 15 300-319 15 16 16 23 27 27 F T T <---
18952024-02-04T14:01:01.756Z 16 320-339 15 15 15 23 23 23 F F F
18962024-02-04T14:01:01.756Z 17 340-359 3 16 16 6 24 24 T F F <---
18972024-02-04T14:01:01.756Z 18 360-379 15 15 15 23 23 23 F F F
18982024-02-04T14:01:01.756Z 19 380-399 15 15 15 23 23 23 F F F
18992024-02-04T14:01:01.756Z 20 400-419 12 16 16 20 24 24 F F F <---
19002024-02-04T14:01:01.756Z 21 420-439 15 15 15 23 23 23 F F F
19012024-02-04T14:01:01.756Z 22 440-459 15 15 15 23 23 23 F F F
19022024-02-04T14:01:01.756Z 23 460-479 12 12 12 20 20 20 F F F
19032024-02-04T14:01:01.756Z 24 480-499 12 12 12 20 20 20 F F F
19042024-02-04T14:01:01.756Z 25 500-519 15 16 16 23 25 25 F F F <---
19052024-02-04T14:01:01.756Z 26 520-539 6 6 6 12 12 12 F F F
19062024-02-04T14:01:01.756Z 27 540-559 15 15 15 23 23 23 F F F
19072024-02-04T14:01:01.756Z 28 560-579 5 16 16 9 24 24 F F F <---
19082024-02-04T14:01:01.757Z 29 580-599 11 11 11 19 19 19 F F F
19092024-02-04T14:01:01.757ZMax gen: 16, Max flush: 27
19102024-02-04T14:01:01.757ZError: Difference in extent metadata found!
19112024-02-04T14:01:01.761ZA Difference in extent metadata is expected here
19122024-02-04T14:01:01.761ZOn loop 8
19132024-02-04T14:01:01.761Z
19142024-02-04T14:01:01.761ZVerifying data now
19152024-02-04T14:01:07.777ZNo change after verify
19162024-02-04T14:01:07.777ZLoop: 8 Downstairs dump after verify (and repair):
19172024-02-04T14:01:07.801ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19182024-02-04T14:01:07.801Z 0 000-019 2 2 2 3 3 3 F F F
19192024-02-04T14:01:07.801Z 1 020-039 16 16 16 26 26 26 F F F
19202024-02-04T14:01:07.801Z 2 040-059 15 15 15 23 23 23 F F F
19212024-02-04T14:01:07.801Z 3 060-079 15 15 15 23 23 23 F F F
19222024-02-04T14:01:07.801Z 4 080-099 7 7 7 14 14 14 F F F
19232024-02-04T14:01:07.801Z 5 100-119 14 14 14 22 22 22 F F F
19242024-02-04T14:01:07.801Z 6 120-139 14 14 14 22 22 22 F F F
19252024-02-04T14:01:07.801Z 7 140-159 1 1 1 1 1 1 F F F
19262024-02-04T14:01:07.801Z 8 160-179 16 16 16 24 24 24 F F F
19272024-02-04T14:01:07.801Z 9 180-199 17 17 17 28 28 28 F F F
19282024-02-04T14:01:07.805Z 10 200-219 17 17 17 28 28 28 F F F
19292024-02-04T14:01:07.805Z 11 220-239 13 13 13 21 21 21 F F F
19302024-02-04T14:01:07.805Z 12 240-259 12 12 12 20 20 20 F F F
19312024-02-04T14:01:07.805Z 13 260-279 16 16 16 24 24 24 F F F
19322024-02-04T14:01:07.805Z 14 280-299 16 16 16 24 24 24 F F F
19332024-02-04T14:01:07.805Z 15 300-319 17 17 17 28 28 28 F F F
19342024-02-04T14:01:07.805Z 16 320-339 15 15 15 23 23 23 F F F
19352024-02-04T14:01:07.805Z 17 340-359 16 16 16 24 24 24 F F F
19362024-02-04T14:01:07.805Z 18 360-379 15 15 15 23 23 23 F F F
19372024-02-04T14:01:07.805Z 19 380-399 15 15 15 23 23 23 F F F
19382024-02-04T14:01:07.805Z 20 400-419 16 16 16 24 24 24 F F F
19392024-02-04T14:01:07.805Z 21 420-439 15 15 15 23 23 23 F F F
19402024-02-04T14:01:07.806Z 22 440-459 15 15 15 23 23 23 F F F
19412024-02-04T14:01:07.806Z 23 460-479 12 12 12 20 20 20 F F F
19422024-02-04T14:01:07.806Z 24 480-499 12 12 12 20 20 20 F F F
19432024-02-04T14:01:07.806Z 25 500-519 16 16 16 25 25 25 F F F
19442024-02-04T14:01:07.806Z 26 520-539 6 6 6 12 12 12 F F F
19452024-02-04T14:01:07.806Z 27 540-559 15 15 15 23 23 23 F F F
19462024-02-04T14:01:07.806Z 28 560-579 16 16 16 24 24 24 F F F
19472024-02-04T14:01:07.806Z 29 580-599 11 11 11 19 19 19 F F F
19482024-02-04T14:01:07.806ZMax gen: 17, Max flush: 28
19492024-02-04T14:01:07.809Z
19502024-02-04T14:01:07.809ZBegin loop 9
19512024-02-04T14:01:07.809ZDownstairs to restart: 1
19522024-02-04T14:01:07.839Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.872146327Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201}
19532024-02-04T14:01:07.839Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.872652924Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201}
19542024-02-04T14:01:07.843Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.87272709Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201}
19552024-02-04T14:01:07.843Z{Crucible runtime is spawned
19562024-02-04T14:01:07.843Z"msg":"Crucible a8a503a1-66fe-4623-89c3-8140a59f8647 has session id: 1b86796a-107a-438f-97fe-04bc8cfeb5e6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.872952282Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19572024-02-04T14:01:07.843Z{"msg":"Upstairs opts: Upstairs UUID: a8a503a1-66fe-4623-89c3-8140a59f8647, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873050248Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19582024-02-04T14:01:07.843Z{"msg":"Crucible stats registered with UUID: a8a503a1-66fe-4623-89c3-8140a59f8647","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.87310675Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19592024-02-04T14:01:07.843Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873130349Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201}
19602024-02-04T14:01:07.843Z{"msg":"a8a503a1-66fe-4623-89c3-8140a59f8647 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873346399Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19612024-02-04T14:01:07.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873430929Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19622024-02-04T14:01:07.843Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873475288Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19632024-02-04T14:01:07.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873509285Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19642024-02-04T14:01:07.843Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873584276Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19652024-02-04T14:01:07.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873611335Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19662024-02-04T14:01:07.843Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873675204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19672024-02-04T14:01:07.843Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873737096Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19682024-02-04T14:01:07.843Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.87381653Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19692024-02-04T14:01:07.844Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873918825Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19702024-02-04T14:01:07.844Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.873992565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19712024-02-04T14:01:07.844Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.874127443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19722024-02-04T14:01:07.844Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.874173172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19732024-02-04T14:01:07.844Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.875139115Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19742024-02-04T14:01:07.844Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.875197146Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19752024-02-04T14:01:07.844Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.875220578Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19762024-02-04T14:01:07.844Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.875242991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19772024-02-04T14:01:07.844Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.8760593Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19782024-02-04T14:01:07.844Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.876307042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19792024-02-04T14:01:07.848Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.87696146Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19802024-02-04T14:01:07.848Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.877030079Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19812024-02-04T14:01:07.848Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.877075605Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19822024-02-04T14:01:07.848Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.877334188Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19832024-02-04T14:01:07.848Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.877399792Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19842024-02-04T14:01:07.848Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.877426613Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19852024-02-04T14:01:07.848Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.877467856Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19862024-02-04T14:01:07.848Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.877537445Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19872024-02-04T14:01:07.848Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.87826298Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19882024-02-04T14:01:07.849Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.878954255Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19892024-02-04T14:01:07.849Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.87903275Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19902024-02-04T14:01:07.849Z{"msg":"[0]R flush_numbers[0..12]: [3, 26, 23, 23, 14, 22, 22, 1, 24, 28, 28, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879110313Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19912024-02-04T14:01:07.849Z{"msg":"[0]R generation[0..12]: [2, 16, 15, 15, 7, 14, 14, 1, 16, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879182178Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19922024-02-04T14:01:07.849Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879258235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19932024-02-04T14:01:07.849Z{"msg":"[1]R flush_numbers[0..12]: [3, 26, 23, 23, 14, 22, 22, 1, 24, 28, 28, 21]","v":0,"Wait for a query_work_queue command to finish before sending IO
19942024-02-04T14:01:07.849Zname":"crucible","level":30,"time":"2024-02-04T14:01:06.879418661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19952024-02-04T14:01:07.849Z{"msg":"[1]R generation[0..12]: [2, 16, 15, 15, 7, 14, 14, 1, 16, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879493981Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19962024-02-04T14:01:07.849Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879568686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19972024-02-04T14:01:07.849Z{"msg":"[2]R flush_numbers[0..12]: [3, 26, 23, 23, 14, 22, 22, 1, 24, 28, 28, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879639572Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19982024-02-04T14:01:07.849Z{"msg":"[2]R generation[0..12]: [2, 16, 15, 15, 7, 14, 14, 1, 16, 17, 17, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879687085Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
19992024-02-04T14:01:07.849Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879761682Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20002024-02-04T14:01:07.849Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879834442Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20012024-02-04T14:01:07.849Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879907202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20022024-02-04T14:01:07.849Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.879974209Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20032024-02-04T14:01:07.849Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880064109Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20042024-02-04T14:01:07.849Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880135482Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20052024-02-04T14:01:07.849Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880202529Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20062024-02-04T14:01:07.849Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880249945Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20072024-02-04T14:01:07.849Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20082024-02-04T14:01:07.849Z,"time":"2024-02-04T14:01:06.880318685Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20092024-02-04T14:01:07.849Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880460897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20102024-02-04T14:01:07.849Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880489762Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20112024-02-04T14:01:07.849Z{"msg":"a8a503a1-66fe-4623-89c3-8140a59f8647 is now active with session: 1b86796a-107a-438f-97fe-04bc8cfeb5e6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880532076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20122024-02-04T14:01:07.850Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880553609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201,"session_id":"1b86796a-107a-438f-97fe-04bc8cfeb5e6"}
20132024-02-04T14:01:07.850Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:06.880575586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201}
20142024-02-04T14:01:07.850ZLoading write count information from file "/tmp/test_repair_verify.data"
20152024-02-04T14:01:07.850ZRun Repair workload
20162024-02-04T14:01:07.853Z01/30 Write block 211 len 28672 data: 2 2 2 2 3 5 5
20172024-02-04T14:01:07.853Z02/30 Flush
20182024-02-04T14:01:07.856Z03/30 Write block 99 len 12288 data: 2 2 2
20192024-02-04T14:01:07.856Z04/30 Read block 123 len 40960
20202024-02-04T14:01:07.859Z05/30 Read block 295 len 32768
20212024-02-04T14:01:07.859Z06/30 Read block 545 len 16384
20222024-02-04T14:01:07.861Z07/30 Read block 506 len 40960
20232024-02-04T14:01:07.862Z08/30 Read block 390 len 16384
20242024-02-04T14:01:07.864Z09/30 Flush
20252024-02-04T14:01:07.865Z10/30 Flush
20262024-02-04T14:01:07.870Z11/30 Write block 316 len 16384 data: 2 3 3 3
20272024-02-04T14:01:07.870Z12/30 Read block 443 len 16384
20282024-02-04T14:01:07.873Z13/30 Write block 168 len 32768 data: 3 4 5 5 6 6 7 5
20292024-02-04T14:01:07.873Z14/30 Read block 133 len 8192
20302024-02-04T14:01:07.873Z15/30 Read block 545 len 28672
20312024-02-04T14:01:07.873Z16/30 Flush
20322024-02-04T14:01:07.876Z17/30 Write block 241 len 40960 data: 2 2 3 3 3 3 3 3 4 4
20332024-02-04T14:01:07.879Z18/30 Write block 479 len 12288 data: 2 2 2
20342024-02-04T14:01:07.879Z19/30 Read block 405 len 36864
20352024-02-04T14:01:07.882Z20/30 Flush
20362024-02-04T14:01:07.885Z21/30 Write block 140 len 28672 data: 2 2 2 2 2 2 2
20372024-02-04T14:01:07.885Z22/30 Read block 455 len 28672
20382024-02-04T14:01:07.888Z23/30 Write block 540 len 8192 data: 2 3
20392024-02-04T14:01:07.888Z24/30 Flush
20402024-02-04T14:01:07.891Z25/30 Write block 164 len 12288 data: 2 2 2
20412024-02-04T14:01:07.894Z26/30 Read block 329 len 32768
20422024-02-04T14:01:07.894Z27/30 Write block 35 len 12288 data: 3 3 3
20432024-02-04T14:01:07.894Z28/30 Read block 392 len 8192
20442024-02-04T14:01:07.897Z29/30 Write block 179 len 40960 data: 5 5 5 4 4 4 4 3 3 3
20452024-02-04T14:01:07.897Z30/30 Write block 543 len 4096 data: 3
20462024-02-04T14:01:07.897Z----------------------------------------------------------------
20472024-02-04T14:01:07.897Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:28
20482024-02-04T14:01:07.897ZStates: Active Active Active
20492024-02-04T14:01:07.897ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20502024-02-04T14:01:07.898Z 3 Acked 1002 Write 3 Done Done Done false
20512024-02-04T14:01:07.898Z 4 Acked 1003 Read 10 Done Done Done false
20522024-02-04T14:01:07.898Z 5 Acked 1004 Read 8 Done Sent Done false
20532024-02-04T14:01:07.898Z 6 Acked 1005 Read 4 Done Sent Done false
20542024-02-04T14:01:07.898Z 7 Acked 1006 Read 10 Done Sent Done false
20552024-02-04T14:01:07.898Z 8 Acked 1007 Read 4 Done Sent Done false
20562024-02-04T14:01:07.898Z 9 Acked 1008 Flush 0 Done Sent Done false
20572024-02-04T14:01:07.898Z 10 Acked 1009 Flush 0 Done Sent Done false
20582024-02-04T14:01:07.898Z 11 Acked 1010 Write 4 Done Sent Done false
20592024-02-04T14:01:07.898Z 12 Acked 1011 Read 4 Done Sent Done false
20602024-02-04T14:01:07.898Z 13 Acked 1012 Write 8 Done Sent Done false
20612024-02-04T14:01:07.898Z 14 Acked 1013 Read 2 Done Sent Done false
20622024-02-04T14:01:07.898Z 15 Acked 1014 Read 7 Done Sent Done false
20632024-02-04T14:01:07.898Z 16 Acked 1015 Flush 0 Done Sent Done false
20642024-02-04T14:01:07.898Z 17 Acked 1016 Write 10 Done Sent Done false
20652024-02-04T14:01:07.898Z 18 Acked 1017 Write 3 Done Sent Done false
20662024-02-04T14:01:07.898Z 19 Acked 1018 Read 9 Done Sent Done false
20672024-02-04T14:01:07.898Z 20 Acked 1019 Flush 0 Done Sent Done false
20682024-02-04T14:01:07.898Z 21 Acked 1020 Write 7 Done Sent Done false
20692024-02-04T14:01:07.898Z 22 Acked 1021 Read 7 Done Sent Done false
20702024-02-04T14:01:07.898Z 23 Acked 1022 Write 2 Done Sent Done false
20712024-02-04T14:01:07.898Z 24 Acked 1023 Flush 0 Done Sent Done false
20722024-02-04T14:01:07.898Z 25 Acked 1024 Write 3 Done Sent Done false
20732024-02-04T14:01:07.898Z 26 Acked 1025 Read 8 Done Sent Done false
20742024-02-04T14:01:07.899Z 27 Acked 1026 Write 3 Done Sent Done false
20752024-02-04T14:01:07.899Z 28 Acked 1027 Read 2 Done Sent Done false
20762024-02-04T14:01:07.899Z 29 Acked 1028 Write 10 Sent Sent Sent false
20772024-02-04T14:01:07.899Z 30 Acked 1029 Write 1 Sent Sent Sent false
20782024-02-04T14:01:07.899Z STATES DS:0 DS:1 DS:2 TOTAL
20792024-02-04T14:01:07.899Z New 0 0 0 0
20802024-02-04T14:01:07.899Z Sent 2 26 2 30
20812024-02-04T14:01:07.899Z Done 26 2 26 54
20822024-02-04T14:01:07.899Z Skipped 0 0 0 0
20832024-02-04T14:01:07.899Z Error 0 0 0 0
20842024-02-04T14:01:07.899ZLast Flush: 1023 1001 1023
20852024-02-04T14:01:07.899ZDownstairs last five completed: 1001 1000
20862024-02-04T14:01:07.899ZUpstairs last five completed: 30 29 28 27 26
20872024-02-04T14:01:07.902Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:06.931324986Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1201}
20882024-02-04T14:01:07.902ZWrote out file "/tmp/test_repair_verify.data"
20892024-02-04T14:01:07.905Z
20902024-02-04T14:01:07.912ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20912024-02-04T14:01:07.938ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20922024-02-04T14:01:07.938Z 0 000-019 2 2 2 3 3 3 F F F
20932024-02-04T14:01:07.938Z 1 020-039 16 16 16 26 26 26 T F T <---
20942024-02-04T14:01:07.938Z 2 040-059 15 15 15 23 23 23 F F F
20952024-02-04T14:01:07.938Z 3 060-079 15 15 15 23 23 23 F F F
20962024-02-04T14:01:07.938Z 4 080-099 18 7 18 30 14 30 F T F <---
20972024-02-04T14:01:07.939Z 5 100-119 18 14 18 30 22 30 F T F <---
20982024-02-04T14:01:07.939Z 6 120-139 14 14 14 22 22 22 F F F
20992024-02-04T14:01:07.939Z 7 140-159 18 1 18 34 1 34 F F F <---
21002024-02-04T14:01:07.939Z 8 160-179 18 16 18 32 24 32 T F T <---
21012024-02-04T14:01:07.939Z 9 180-199 17 17 17 28 28 28 T F T <---
21022024-02-04T14:01:07.939Z 10 200-219 18 18 18 29 29 29 F F F
21032024-02-04T14:01:07.939Z 11 220-239 13 13 13 21 21 21 F F F
21042024-02-04T14:01:07.939Z 12 240-259 18 12 18 33 20 33 F F F <---
21052024-02-04T14:01:07.939Z 13 260-279 16 16 16 24 24 24 F F F
21062024-02-04T14:01:07.939Z 14 280-299 16 16 16 24 24 24 F F F
21072024-02-04T14:01:07.942Z 15 300-319 18 17 18 32 28 32 F F F <---
21082024-02-04T14:01:07.942Z 16 320-339 15 15 15 23 23 23 F F F
21092024-02-04T14:01:07.942Z 17 340-359 16 16 16 24 24 24 F F F
21102024-02-04T14:01:07.942Z 18 360-379 15 15 15 23 23 23 F F F
21112024-02-04T14:01:07.942Z 19 380-399 15 15 15 23 23 23 F F F
21122024-02-04T14:01:07.942Z 20 400-419 16 16 16 24 24 24 F F F
21132024-02-04T14:01:07.942Z 21 420-439 15 15 15 23 23 23 F F F
21142024-02-04T14:01:07.942Z 22 440-459 15 15 15 23 23 23 F F F
21152024-02-04T14:01:07.942Z 23 460-479 18 12 18 33 20 33 F F F <---
21162024-02-04T14:01:07.942Z 24 480-499 18 12 18 33 20 33 F F F <---
21172024-02-04T14:01:07.942Z 25 500-519 16 16 16 25 25 25 F F F
21182024-02-04T14:01:07.943Z 26 520-539 6 6 6 12 12 12 F F F
21192024-02-04T14:01:07.943Z 27 540-559 18 15 18 34 23 34 T F T <---
21202024-02-04T14:01:07.943Z 28 560-579 16 16 16 24 24 24 F F F
21212024-02-04T14:01:07.943Z 29 580-599 11 11 11 19 19 19 F F F
21222024-02-04T14:01:07.943ZMax gen: 18, Max flush: 34
21232024-02-04T14:01:07.943ZError: Difference in extent metadata found!
21242024-02-04T14:01:07.946ZA Difference in extent metadata is expected here
21252024-02-04T14:01:07.946ZOn loop 9
21262024-02-04T14:01:07.946Z
21272024-02-04T14:01:07.951ZVerifying data now
21282024-02-04T14:01:13.936ZNo change after verify
21292024-02-04T14:01:13.936ZLoop: 9 Downstairs dump after verify (and repair):
21302024-02-04T14:01:13.959ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21312024-02-04T14:01:13.960Z 0 000-019 2 2 2 3 3 3 F F F
21322024-02-04T14:01:13.960Z 1 020-039 19 19 19 35 35 35 F F F
21332024-02-04T14:01:13.960Z 2 040-059 15 15 15 23 23 23 F F F
21342024-02-04T14:01:13.960Z 3 060-079 15 15 15 23 23 23 F F F
21352024-02-04T14:01:13.963Z 4 080-099 18 18 18 30 30 30 F F F
21362024-02-04T14:01:13.963Z 5 100-119 18 18 18 30 30 30 F F F
21372024-02-04T14:01:13.963Z 6 120-139 14 14 14 22 22 22 F F F
21382024-02-04T14:01:13.963Z 7 140-159 18 18 18 34 34 34 F F F
21392024-02-04T14:01:13.963Z 8 160-179 19 19 19 35 35 35 F F F
21402024-02-04T14:01:13.963Z 9 180-199 19 19 19 35 35 35 F F F
21412024-02-04T14:01:13.963Z 10 200-219 18 18 18 29 29 29 F F F
21422024-02-04T14:01:13.964Z 11 220-239 13 13 13 21 21 21 F F F
21432024-02-04T14:01:13.964Z 12 240-259 18 18 18 33 33 33 F F F
21442024-02-04T14:01:13.964Z 13 260-279 16 16 16 24 24 24 F F F
21452024-02-04T14:01:13.964Z 14 280-299 16 16 16 24 24 24 F F F
21462024-02-04T14:01:13.964Z 15 300-319 18 18 18 32 32 32 F F F
21472024-02-04T14:01:13.964Z 16 320-339 15 15 15 23 23 23 F F F
21482024-02-04T14:01:13.964Z 17 340-359 16 16 16 24 24 24 F F F
21492024-02-04T14:01:13.964Z 18 360-379 15 15 15 23 23 23 F F F
21502024-02-04T14:01:13.964Z 19 380-399 15 15 15 23 23 23 F F F
21512024-02-04T14:01:13.964Z 20 400-419 16 16 16 24 24 24 F F F
21522024-02-04T14:01:13.964Z 21 420-439 15 15 15 23 23 23 F F F
21532024-02-04T14:01:13.964Z 22 440-459 15 15 15 23 23 23 F F F
21542024-02-04T14:01:13.964Z 23 460-479 18 18 18 33 33 33 F F F
21552024-02-04T14:01:13.964Z 24 480-499 18 18 18 33 33 33 F F F
21562024-02-04T14:01:13.964Z 25 500-519 16 16 16 25 25 25 F F F
21572024-02-04T14:01:13.964Z 26 520-539 6 6 6 12 12 12 F F F
21582024-02-04T14:01:13.964Z 27 540-559 19 19 19 35 35 35 F F F
21592024-02-04T14:01:13.964Z 28 560-579 16 16 16 24 24 24 F F F
21602024-02-04T14:01:13.964Z 29 580-599 11 11 11 19 19 19 F F F
21612024-02-04T14:01:13.964ZMax gen: 19, Max flush: 35
21622024-02-04T14:01:13.967Z
21632024-02-04T14:01:13.967ZBegin loop 10
21642024-02-04T14:01:13.968ZDownstairs to restart: 1
21652024-02-04T14:01:13.998Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.031338619Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209}
21662024-02-04T14:01:13.998Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.031826702Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209}
21672024-02-04T14:01:14.002Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.031894979Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209}
21682024-02-04T14:01:14.002Z{"msg":"Crucible f9f2e5c8-6ab7-40fb-8237-8ee41ba7b659 has session id: 843fa091-55b9-44c1-80e5-f68b9c56b1a4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.031951809Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21692024-02-04T14:01:14.002Z{"msg":"Crucible runtime is spawned
21702024-02-04T14:01:14.002ZUpstairs opts: Upstairs UUID: f9f2e5c8-6ab7-40fb-8237-8ee41ba7b659, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.0320225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21712024-02-04T14:01:14.002Z{"msg":"Crucible stats registered with UUID: f9f2e5c8-6ab7-40fb-8237-8ee41ba7b659","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.032072339Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21722024-02-04T14:01:14.002Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.032163234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209}
21732024-02-04T14:01:14.002Z{"msg":"f9f2e5c8-6ab7-40fb-8237-8ee41ba7b659 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.032375416Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21742024-02-04T14:01:14.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.032430765Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21752024-02-04T14:01:14.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.03247306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21762024-02-04T14:01:14.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.032512283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21772024-02-04T14:01:14.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.032574328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21782024-02-04T14:01:14.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.032664441Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21792024-02-04T14:01:14.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.032780616Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21802024-02-04T14:01:14.002Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.03285349Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21812024-02-04T14:01:14.003Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.033127486Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21822024-02-04T14:01:14.003Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.033210971Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21832024-02-04T14:01:14.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.033255795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21842024-02-04T14:01:14.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.033329884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21852024-02-04T14:01:14.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.033372887Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21862024-02-04T14:01:14.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.034475621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21872024-02-04T14:01:14.003Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.034552697Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21882024-02-04T14:01:14.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.034664805Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21892024-02-04T14:01:14.003Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.034754718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21902024-02-04T14:01:14.007Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.035469437Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21912024-02-04T14:01:14.007Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.035775281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21922024-02-04T14:01:14.007Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.03598501Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21932024-02-04T14:01:14.007Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.036020946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21942024-02-04T14:01:14.007Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.036454217Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21952024-02-04T14:01:14.007Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.036529336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21962024-02-04T14:01:14.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.036612462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21972024-02-04T14:01:14.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.036668484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21982024-02-04T14:01:14.008Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.036695329Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
21992024-02-04T14:01:14.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.0367179Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22002024-02-04T14:01:14.008Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.036968351Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22012024-02-04T14:01:14.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.03767297Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22022024-02-04T14:01:14.008Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.037745859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22032024-02-04T14:01:14.008Z{"msg":"[0]R flush_numbers[0..12]: [3, 35, 23, 23, 30, 30, 22, 34, 35, 35, 29, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.037896497Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22042024-02-04T14:01:14.008Z{"msg":"[0]R generation[0..12]: [2, 19, 15, 15, 18, 18, 14, 18, 19, 19, 18, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.037950807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22052024-02-04T14:01:14.008Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038046586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22062024-02-04T14:01:14.008Z{"msg":"[1]R flush_numbers[0..12]: [3, 35, 23, 23, 30, 30, 22, 34, 35, 35, 29, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038094639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22072024-02-04T14:01:14.008Z{"msg":"[1]R generation[0..12]: [2, 19, 15, 15, 18, 18, 14, 18, 19, 19, 18, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038119676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22082024-02-04T14:01:14.008Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO
22092024-02-04T14:01:14.008Z:30,"time":"2024-02-04T14:01:13.038147426Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22102024-02-04T14:01:14.008Z{"msg":"[2]R flush_numbers[0..12]: [3, 35, 23, 23, 30, 30, 22, 34, 35, 35, 29, 21]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038171304Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22112024-02-04T14:01:14.008Z{"msg":"[2]R generation[0..12]: [2, 19, 15, 15, 18, 18, 14, 18, 19, 19, 18, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038212339Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22122024-02-04T14:01:14.008Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038237895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22132024-02-04T14:01:14.008Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038260843Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22142024-02-04T14:01:14.008Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038304936Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22152024-02-04T14:01:14.008Z{"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038328066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22162024-02-04T14:01:14.008Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038350259Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22172024-02-04T14:01:14.008Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038392188Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22182024-02-04T14:01:14.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038413884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22192024-02-04T14:01:14.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038498357Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22202024-02-04T14:01:14.009Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038522397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22212024-02-04T14:01:14.009Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038575259Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22222024-02-04T14:01:14.009Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038599055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22232024-02-04T14:01:14.009Z{"msg":"f9f2e5c8-6ab7-40fb-8237-8ee41ba7b659 is now active with session: 843fa091-55b9-44c1-80e5-f68b9c56b1a4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038627661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22242024-02-04T14:01:14.009Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038668935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209,"session_id":"843fa091-55b9-44c1-80e5-f68b9c56b1a4"}
22252024-02-04T14:01:14.009Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:13.038691835Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209}
22262024-02-04T14:01:14.009ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22272024-02-04T14:01:14.009ZLoading write count information from file "/tmp/test_repair_verify.data"
22282024-02-04T14:01:14.009ZRun Repair workload
22292024-02-04T14:01:14.009Z01/30 Write block 533 len 16384 data: 2 2 2 2
22302024-02-04T14:01:14.012Z02/30 Write block 237 len 20480 data: 2 2 2 2 3
22312024-02-04T14:01:14.012Z03/30 Write block 250 len 12288 data: 5 4 4
22322024-02-04T14:01:14.012Z04/30 Read block 221 len 12288
22332024-02-04T14:01:14.015Z05/30 Write block 111 len 24576 data: 5 4 5 4 4 4
22342024-02-04T14:01:14.015Z06/30 Write block 341 len 28672 data: 2 2 2 2 2 2 2
22352024-02-04T14:01:14.015Z07/30 Read block 237 len 12288
22362024-02-04T14:01:14.018Z08/30 Read block 452 len 8192
22372024-02-04T14:01:14.018Z09/30 Write block 278 len 16384 data: 3 3 3 3
22382024-02-04T14:01:14.024Z10/30 Write block 84 len 32768 data: 2 2 2 2 2 2 2 2
22392024-02-04T14:01:14.024Z11/30 Read block 230 len 24576
22402024-02-04T14:01:14.027Z12/30 Read block 161 len 40960
22412024-02-04T14:01:14.027Z13/30 Write block 196 len 36864 data: 2 2 3 4 4 4 3 3 3
22422024-02-04T14:01:14.027Z14/30 Read block 306 len 32768
22432024-02-04T14:01:14.030Z15/30 Read block 9 len 28672
22442024-02-04T14:01:14.030Z16/30 Write block 112 len 32768 data: 5 6 5 5 5 5 3 3
22452024-02-04T14:01:14.032Z17/30 Read block 495 len 40960
22462024-02-04T14:01:14.035Z18/30 Read block 333 len 28672
22472024-02-04T14:01:14.035Z19/30 Read block 261 len 32768
22482024-02-04T14:01:14.038Z20/30 Read block 587 len 8192
22492024-02-04T14:01:14.038Z21/30 Write block 420 len 4096 data: 4
22502024-02-04T14:01:14.038Z22/30 Read block 202 len 32768
22512024-02-04T14:01:14.041Z23/30 Read block 556 len 4096
22522024-02-04T14:01:14.041Z24/30 Read block 311 len 40960
22532024-02-04T14:01:14.044Z25/30 Read block 203 len 16384
22542024-02-04T14:01:14.044Z26/30 Read block 151 len 40960
22552024-02-04T14:01:14.047Z27/30 Write block 365 len 32768 data: 3 3 3 3 3 3 2 2
22562024-02-04T14:01:14.047Z28/30 Read block 495 len 12288
22572024-02-04T14:01:14.050Z29/30 Write block 134 len 8192 data: 2 2
22582024-02-04T14:01:14.050Z30/30 Read block 383 len 40960
22592024-02-04T14:01:14.053Z----------------------------------------------------------------
22602024-02-04T14:01:14.053Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22612024-02-04T14:01:14.053ZStates: Active Active Active
22622024-02-04T14:01:14.053ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22632024-02-04T14:01:14.053Z 1 Acked 1000 Write 4 Done Done Done false
22642024-02-04T14:01:14.053Z 2 Acked 1001 Write 5 Done Done Done false
22652024-02-04T14:01:14.053Z 3 Acked 1002 Write 3 Done Sent Done false
22662024-02-04T14:01:14.053Z 4 Acked 1003 Read 3 Done Sent Done false
22672024-02-04T14:01:14.053Z 5 Acked 1004 Write 6 Done Sent Done false
22682024-02-04T14:01:14.053Z 6 Acked 1005 Write 7 Done Sent Done false
22692024-02-04T14:01:14.053Z 7 Acked 1006 Read 3 Done Sent Done false
22702024-02-04T14:01:14.054Z 8 Acked 1007 Read 2 Done Sent Done false
22712024-02-04T14:01:14.054Z 9 Acked 1008 Write 4 Done Sent Done false
22722024-02-04T14:01:14.054Z 10 Acked 1009 Write 8 Done Sent Done false
22732024-02-04T14:01:14.054Z 11 Acked 1010 Read 6 Done Sent Done false
22742024-02-04T14:01:14.054Z 12 Acked 1011 Read 10 Done Sent Done false
22752024-02-04T14:01:14.054Z 13 Acked 1012 Write 9 Done Sent Done false
22762024-02-04T14:01:14.054Z 14 Acked 1013 Read 8 Done Sent Done false
22772024-02-04T14:01:14.054Z 15 Acked 1014 Read 7 Done Sent Done false
22782024-02-04T14:01:14.054Z 16 Acked 1015 Write 8 Done Sent Done false
22792024-02-04T14:01:14.054Z 17 Acked 1016 Read 10 Done Sent Done false
22802024-02-04T14:01:14.054Z 18 Acked 1017 Read 7 Done Sent Done false
22812024-02-04T14:01:14.054Z 19 Acked 1018 Read 8 Done Sent Done false
22822024-02-04T14:01:14.054Z 20 Acked 1019 Read 2 Done Sent Done false
22832024-02-04T14:01:14.054Z 21 Acked 1020 Write 1 Done Sent Done false
22842024-02-04T14:01:14.054Z 22 Acked 1021 Read 8 Done Sent Done false
22852024-02-04T14:01:14.054Z 23 Acked 1022 Read 1 Done Sent Done false
22862024-02-04T14:01:14.054Z 24 Acked 1023 Read 10 Done Sent Done false
22872024-02-04T14:01:14.054Z 25 Acked 1024 Read 4 Done Sent Done false
22882024-02-04T14:01:14.054Z 26 Acked 1025 Read 10 Done Sent Done false
22892024-02-04T14:01:14.054Z 27 Acked 1026 Write 8 Done Sent Done false
22902024-02-04T14:01:14.054Z 28 Acked 1027 Read 3 Done Sent Done false
22912024-02-04T14:01:14.054Z 29 Acked 1028 Write 2 Done Sent Done false
22922024-02-04T14:01:14.054Z 30 Acked 1029 Read 10 Done Sent Sent false
22932024-02-04T14:01:14.054Z STATES DS:0 DS:1 DS:2 TOTAL
22942024-02-04T14:01:14.055Z New 0 0 0 0
22952024-02-04T14:01:14.055Z Sent 0 28 1 29
22962024-02-04T14:01:14.055Z Done 30 2 29 61
22972024-02-04T14:01:14.055Z Skipped 0 0 0 0
22982024-02-04T14:01:14.055Z Error 0 0 0 0
22992024-02-04T14:01:14.055ZLast Flush: 0 0 0
23002024-02-04T14:01:14.055ZDownstairs last five completed:
23012024-02-04T14:01:14.055ZUpstairs last five completed: 30 29 28 27 26
23022024-02-04T14:01:14.055Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:13.085226421Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1209}
23032024-02-04T14:01:14.055ZWrote out file "/tmp/test_repair_verify.data"
23042024-02-04T14:01:14.059Z
23052024-02-04T14:01:14.066ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23062024-02-04T14:01:14.098ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23072024-02-04T14:01:14.098Z 0 000-019 2 2 2 3 3 3 F F F
23082024-02-04T14:01:14.098Z 1 020-039 19 19 19 35 35 35 F F F
23092024-02-04T14:01:14.098Z 2 040-059 15 15 15 23 23 23 F F F
23102024-02-04T14:01:14.098Z 3 060-079 15 15 15 23 23 23 F F F
23112024-02-04T14:01:14.098Z 4 080-099 18 18 18 30 30 30 T F T <---
23122024-02-04T14:01:14.098Z 5 100-119 18 18 18 30 30 30 T F T <---
23132024-02-04T14:01:14.098Z 6 120-139 14 14 14 22 22 22 T F T <---
23142024-02-04T14:01:14.098Z 7 140-159 18 18 18 34 34 34 F F F
23152024-02-04T14:01:14.098Z 8 160-179 19 19 19 35 35 35 F F F
23162024-02-04T14:01:14.098Z 9 180-199 19 19 19 35 35 35 T F T <---
23172024-02-04T14:01:14.098Z 10 200-219 18 18 18 29 29 29 T F T <---
23182024-02-04T14:01:14.098Z 11 220-239 13 13 13 21 21 21 T T T
23192024-02-04T14:01:14.098Z 12 240-259 18 18 18 33 33 33 T T T
23202024-02-04T14:01:14.098Z 13 260-279 16 16 16 24 24 24 T F T <---
23212024-02-04T14:01:14.098Z 14 280-299 16 16 16 24 24 24 T F T <---
23222024-02-04T14:01:14.098Z 15 300-319 18 18 18 32 32 32 F F F
23232024-02-04T14:01:14.098Z 16 320-339 15 15 15 23 23 23 F F F
23242024-02-04T14:01:14.098Z 17 340-359 16 16 16 24 24 24 T F T <---
23252024-02-04T14:01:14.098Z 18 360-379 15 15 15 23 23 23 T F T <---
23262024-02-04T14:01:14.102Z 19 380-399 15 15 15 23 23 23 F F F
23272024-02-04T14:01:14.102Z 20 400-419 16 16 16 24 24 24 F F F
23282024-02-04T14:01:14.102Z 21 420-439 15 15 15 23 23 23 T F T <---
23292024-02-04T14:01:14.102Z 22 440-459 15 15 15 23 23 23 F F F
23302024-02-04T14:01:14.102Z 23 460-479 18 18 18 33 33 33 F F F
23312024-02-04T14:01:14.102Z 24 480-499 18 18 18 33 33 33 F F F
23322024-02-04T14:01:14.102Z 25 500-519 16 16 16 25 25 25 F F F
23332024-02-04T14:01:14.102Z 26 520-539 6 6 6 12 12 12 T T T
23342024-02-04T14:01:14.102Z 27 540-559 19 19 19 35 35 35 F F F
23352024-02-04T14:01:14.102Z 28 560-579 16 16 16 24 24 24 F F F
23362024-02-04T14:01:14.102Z 29 580-599 11 11 11 19 19 19 F F F
23372024-02-04T14:01:14.102ZMax gen: 19, Max flush: 35
23382024-02-04T14:01:14.102ZError: Difference in extent metadata found!
23392024-02-04T14:01:14.105ZA Difference in extent metadata is expected here
23402024-02-04T14:01:14.105ZOn loop 10
23412024-02-04T14:01:14.105Z
23422024-02-04T14:01:14.108ZVerifying data now
23432024-02-04T14:01:20.296ZNo change after verify
23442024-02-04T14:01:20.296ZLoop: 10 Downstairs dump after verify (and repair):
23452024-02-04T14:01:20.320ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23462024-02-04T14:01:20.320Z 0 000-019 2 2 2 3 3 3 F F F
23472024-02-04T14:01:20.320Z 1 020-039 19 19 19 35 35 35 F F F
23482024-02-04T14:01:20.320Z 2 040-059 15 15 15 23 23 23 F F F
23492024-02-04T14:01:20.320Z 3 060-079 15 15 15 23 23 23 F F F
23502024-02-04T14:01:20.320Z 4 080-099 20 20 20 36 36 36 F F F
23512024-02-04T14:01:20.320Z 5 100-119 20 20 20 36 36 36 F F F
23522024-02-04T14:01:20.320Z 6 120-139 20 20 20 36 36 36 F F F
23532024-02-04T14:01:20.320Z 7 140-159 18 18 18 34 34 34 F F F
23542024-02-04T14:01:20.320Z 8 160-179 19 19 19 35 35 35 F F F
23552024-02-04T14:01:20.324Z 9 180-199 20 20 20 36 36 36 F F F
23562024-02-04T14:01:20.324Z 10 200-219 20 20 20 36 36 36 F F F
23572024-02-04T14:01:20.324Z 11 220-239 20 20 20 36 36 36 F F F
23582024-02-04T14:01:20.325Z 12 240-259 20 20 20 36 36 36 F F F
23592024-02-04T14:01:20.325Z 13 260-279 20 20 20 36 36 36 F F F
23602024-02-04T14:01:20.325Z 14 280-299 20 20 20 36 36 36 F F F
23612024-02-04T14:01:20.325Z 15 300-319 18 18 18 32 32 32 F F F
23622024-02-04T14:01:20.325Z 16 320-339 15 15 15 23 23 23 F F F
23632024-02-04T14:01:20.325Z 17 340-359 20 20 20 36 36 36 F F F
23642024-02-04T14:01:20.325Z 18 360-379 20 20 20 36 36 36 F F F
23652024-02-04T14:01:20.325Z 19 380-399 15 15 15 23 23 23 F F F
23662024-02-04T14:01:20.325Z 20 400-419 16 16 16 24 24 24 F F F
23672024-02-04T14:01:20.325Z 21 420-439 20 20 20 36 36 36 F F F
23682024-02-04T14:01:20.325Z 22 440-459 15 15 15 23 23 23 F F F
23692024-02-04T14:01:20.325Z 23 460-479 18 18 18 33 33 33 F F F
23702024-02-04T14:01:20.326Z 24 480-499 18 18 18 33 33 33 F F F
23712024-02-04T14:01:20.326Z 25 500-519 16 16 16 25 25 25 F F F
23722024-02-04T14:01:20.326Z 26 520-539 20 20 20 36 36 36 F F F
23732024-02-04T14:01:20.326Z 27 540-559 19 19 19 35 35 35 F F F
23742024-02-04T14:01:20.326Z 28 560-579 16 16 16 24 24 24 F F F
23752024-02-04T14:01:20.326Z 29 580-599 11 11 11 19 19 19 F F F
23762024-02-04T14:01:20.326ZMax gen: 20, Max flush: 36
23772024-02-04T14:01:20.330Z
23782024-02-04T14:01:20.330ZBegin loop 11
23792024-02-04T14:01:20.330ZDownstairs to restart: 1
23802024-02-04T14:01:20.357Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.390041134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217}
23812024-02-04T14:01:20.357Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.390514055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217}
23822024-02-04T14:01:20.361Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.39057743Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217}
23832024-02-04T14:01:20.361Z{"msg":"Crucible 519ea489-5c95-4dee-b323-e5ac0627f03e has session id: f40e0ed6-844a-446f-a0ce-8aa118e2dcd3","v":0,"name":"crucible","level":30Crucible runtime is spawned
23842024-02-04T14:01:20.361Z,"time":"2024-02-04T14:01:19.390644703Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23852024-02-04T14:01:20.361Z{"msg":"Upstairs opts: Upstairs UUID: 519ea489-5c95-4dee-b323-e5ac0627f03e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.390702781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23862024-02-04T14:01:20.361Z{"msg":"Crucible stats registered with UUID: 519ea489-5c95-4dee-b323-e5ac0627f03e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.390772775Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23872024-02-04T14:01:20.361Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.390935823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217}
23882024-02-04T14:01:20.361Z{"msg":"519ea489-5c95-4dee-b323-e5ac0627f03e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391194128Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23892024-02-04T14:01:20.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391307527Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23902024-02-04T14:01:20.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391422968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23912024-02-04T14:01:20.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391472217Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23922024-02-04T14:01:20.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391518258Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23932024-02-04T14:01:20.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391561466Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23942024-02-04T14:01:20.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391646967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23952024-02-04T14:01:20.362Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391688723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23962024-02-04T14:01:20.362Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.391866398Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23972024-02-04T14:01:20.362Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.392030808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23982024-02-04T14:01:20.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.39211826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
23992024-02-04T14:01:20.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.392167294Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24002024-02-04T14:01:20.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.392211721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24012024-02-04T14:01:20.362Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.392926054Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24022024-02-04T14:01:20.363Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.393008058Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24032024-02-04T14:01:20.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.393130446Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24042024-02-04T14:01:20.363Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.393208912Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24052024-02-04T14:01:20.363Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.393934202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24062024-02-04T14:01:20.363Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.394039158Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24072024-02-04T14:01:20.367Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.394763204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24082024-02-04T14:01:20.367Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.394827764Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24092024-02-04T14:01:20.368Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.394876754Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24102024-02-04T14:01:20.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.395045441Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24112024-02-04T14:01:20.368Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.395102478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24122024-02-04T14:01:20.368Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.395221157Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24132024-02-04T14:01:20.368Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.395365786Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24142024-02-04T14:01:20.368Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.395394582Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24152024-02-04T14:01:20.368Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.39598871Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24162024-02-04T14:01:20.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.396671944Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24172024-02-04T14:01:20.368Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.396749034Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24182024-02-04T14:01:20.368Z{"msg":"[0]R flush_numbers[0..12]: [3, 35, 23, 23, 36, 36, 36, 34, 35, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.396817491Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24192024-02-04T14:01:20.368Z{"msg":"[0]R generation[0..12]: [2, 19, 15, 15, 20, 20, 20, 18, 19, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.396847482Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24202024-02-04T14:01:20.368Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.396872359Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24212024-02-04T14:01:20.368Z{"msg":"[1]R flush_numbers[0..12]: [3, 35, 23, 23, 36, 36, 36, 34, 35, 36, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.396896061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24222024-02-04T14:01:20.369Z{"msg":"[1]R generation[0..12]: [2, 19, 15, 15, 20, 20, 20, 18, 19, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.39691878Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24232024-02-04T14:01:20.369Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.396942911Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24242024-02-04T14:01:20.369Z{"msg":"[2]R flush_numbers[0..12]: [3, 35, 23, 23, 36, 36, 36, 34, 35, 36, 36, 36]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
24252024-02-04T14:01:20.369Z,"time":"2024-02-04T14:01:19.397003407Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24262024-02-04T14:01:20.369Z{"msg":"[2]R generation[0..12]: [2, 19, 15, 15, 20, 20, 20, 18, 19, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397049735Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24272024-02-04T14:01:20.369Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397075343Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24282024-02-04T14:01:20.369Z{"msg":"Max found gen is 21","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.39712477Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24292024-02-04T14:01:20.369Z{"msg":"Generation requested: 22 >= found:21","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.39714845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24302024-02-04T14:01:20.369Z{"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.39718801Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24312024-02-04T14:01:20.369Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397211134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24322024-02-04T14:01:20.369Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397255957Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24332024-02-04T14:01:20.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397277152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24342024-02-04T14:01:20.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397321064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24352024-02-04T14:01:20.370Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397344389Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24362024-02-04T14:01:20.370Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397366676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24372024-02-04T14:01:20.370Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397387542Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24382024-02-04T14:01:20.370Z{"msg":"519ea489-5c95-4dee-b323-e5ac0627f03e is now active with session: f40e0ed6-844a-446f-a0ce-8aa118e2dcd3","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397409461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24392024-02-04T14:01:20.370Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397464431Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217,"session_id":"f40e0ed6-844a-446f-a0ce-8aa118e2dcd3"}
24402024-02-04T14:01:20.370Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:19.397488621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217}
24412024-02-04T14:01:20.370ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24422024-02-04T14:01:20.370ZLoading write count information from file "/tmp/test_repair_verify.data"
24432024-02-04T14:01:20.370ZRun Repair workload
24442024-02-04T14:01:20.370Z01/30 Write block 159 len 16384 data: 2 2 2 2
24452024-02-04T14:01:20.374Z02/30 Flush
24462024-02-04T14:01:20.374Z03/30 Write block 223 len 8192 data: 3 2
24472024-02-04T14:01:20.374Z04/30 Write block 211 len 40960 data: 3 3 3 3 4 6 6 4 3 3
24482024-02-04T14:01:20.375Z05/30 Read block 537 len 4096
24492024-02-04T14:01:20.378Z06/30 Write block 302 len 8192 data: 4 4
24502024-02-04T14:01:20.378Z07/30 Write block 489 len 32768 data: 4 4 3 3 3 3 3 4
24512024-02-04T14:01:20.378Z08/30 Read block 112 len 20480
24522024-02-04T14:01:20.381Z09/30 Read block 201 len 24576
24532024-02-04T14:01:20.381Z10/30 Read block 22 len 20480
24542024-02-04T14:01:20.384Z11/30 Write block 144 len 40960 data: 3 3 3 2 2 2 2 2 2 2
24552024-02-04T14:01:20.384Z12/30 Write block 112 len 24576 data: 6 7 6 6 6 6
24562024-02-04T14:01:20.384Z13/30 Read block 131 len 8192
24572024-02-04T14:01:20.388Z14/30 Write block 133 len 12288 data: 3 3 3
24582024-02-04T14:01:20.388Z15/30 Flush
24592024-02-04T14:01:20.394Z16/30 Write block 534 len 12288 data: 3 3 3
24602024-02-04T14:01:20.394Z17/30 Read block 120 len 12288
24612024-02-04T14:01:20.397Z18/30 Read block 590 len 32768
24622024-02-04T14:01:20.397Z19/30 Read block 72 len 24576
24632024-02-04T14:01:20.397Z20/30 Write block 126 len 4096 data: 4
24642024-02-04T14:01:20.397Z21/30 Read block 363 len 20480
24652024-02-04T14:01:20.400Z22/30 Write block 199 len 20480 data: 5 5 5 4 4
24662024-02-04T14:01:20.401Z23/30 Read block 521 len 16384
24672024-02-04T14:01:20.401Z24/30 Write block 198 len 4096 data: 4
24682024-02-04T14:01:20.404Z25/30 Write block 576 len 40960 data: 2 2 2 2 2 3 3 3 3 3
24692024-02-04T14:01:20.404Z26/30 Read block 484 len 36864
24702024-02-04T14:01:20.407Z27/30 Write block 474 len 20480 data: 3 2 2 2 2
24712024-02-04T14:01:20.407Z28/30 Write block 287 len 40960 data: 2 3 3 3 3 3 2 2 2 2
24722024-02-04T14:01:20.411Z29/30 Read block 515 len 4096
24732024-02-04T14:01:20.411Z30/30 Read block 26 len 32768
24742024-02-04T14:01:20.415Z----------------------------------------------------------------
24752024-02-04T14:01:20.415Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:28
24762024-02-04T14:01:20.415ZStates: Active Active Active
24772024-02-04T14:01:20.415ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24782024-02-04T14:01:20.415Z 3 Acked 1002 Write 2 Done Done Done false
24792024-02-04T14:01:20.415Z 4 Acked 1003 Write 10 Done Done Done false
24802024-02-04T14:01:20.415Z 5 Acked 1004 Read 1 Done Done Done false
24812024-02-04T14:01:20.415Z 6 Acked 1005 Write 2 Done Done Done false
24822024-02-04T14:01:20.415Z 7 Acked 1006 Write 8 Done Sent Done false
24832024-02-04T14:01:20.416Z 8 Acked 1007 Read 5 Done Sent Done false
24842024-02-04T14:01:20.416Z 9 Acked 1008 Read 6 Done Sent Done false
24852024-02-04T14:01:20.416Z 10 Acked 1009 Read 5 Done Sent Done false
24862024-02-04T14:01:20.416Z 11 Acked 1010 Write 10 Done Sent Done false
24872024-02-04T14:01:20.416Z 12 Acked 1011 Write 6 Done Sent Done false
24882024-02-04T14:01:20.416Z 13 Acked 1012 Read 2 Done Sent Done false
24892024-02-04T14:01:20.416Z 14 Acked 1013 Write 3 Done Sent Done false
24902024-02-04T14:01:20.416Z 15 Acked 1014 Flush 0 Done Sent Done false
24912024-02-04T14:01:20.416Z 16 Acked 1015 Write 3 Done Sent Done false
24922024-02-04T14:01:20.416Z 17 Acked 1016 Read 3 Done Sent Done false
24932024-02-04T14:01:20.416Z 18 Acked 1017 Read 8 Done Sent Done false
24942024-02-04T14:01:20.416Z 19 Acked 1018 Read 6 Done Sent Done false
24952024-02-04T14:01:20.416Z 20 Acked 1019 Write 1 Done Sent Done false
24962024-02-04T14:01:20.417Z 21 Acked 1020 Read 5 Done Sent Done false
24972024-02-04T14:01:20.417Z 22 Acked 1021 Write 5 Done Sent Done false
24982024-02-04T14:01:20.417Z 23 Acked 1022 Read 4 Done Sent Done false
24992024-02-04T14:01:20.417Z 24 Acked 1023 Write 1 Done Sent Done false
25002024-02-04T14:01:20.417Z 25 Acked 1024 Write 10 Done Sent Done false
25012024-02-04T14:01:20.417Z 26 Acked 1025 Read 9 Done Sent Done false
25022024-02-04T14:01:20.417Z 27 Acked 1026 Write 5 Done Sent Done false
25032024-02-04T14:01:20.417Z 28 Acked 1027 Write 10 Done Sent Done false
25042024-02-04T14:01:20.417Z 29 Acked 1028 Read 1 Done Sent Done false
25052024-02-04T14:01:20.417Z 30 Acked 1029 Read 8 Done Sent Done false
25062024-02-04T14:01:20.417Z STATES DS:0 DS:1 DS:2 TOTAL
25072024-02-04T14:01:20.417Z New 0 0 0 0
25082024-02-04T14:01:20.418Z Sent 0 24 0 24
25092024-02-04T14:01:20.418Z Done 28 4 28 60
25102024-02-04T14:01:20.418Z Skipped 0 0 0 0
25112024-02-04T14:01:20.418Z Error 0 0 0 0
25122024-02-04T14:01:20.418ZLast Flush: 1014 1001 1014
25132024-02-04T14:01:20.418ZDownstairs last five completed: 1001 1000
25142024-02-04T14:01:20.418ZUpstairs last five completed: 30 29 28 27 26
25152024-02-04T14:01:20.418Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:19.445331621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1217}
25162024-02-04T14:01:20.418ZWrote out file "/tmp/test_repair_verify.data"
25172024-02-04T14:01:20.422Z
25182024-02-04T14:01:20.426ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25192024-02-04T14:01:20.455ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25202024-02-04T14:01:20.455Z 0 000-019 2 2 2 3 3 3 F F F
25212024-02-04T14:01:20.455Z 1 020-039 19 19 19 35 35 35 F F F
25222024-02-04T14:01:20.455Z 2 040-059 15 15 15 23 23 23 F F F
25232024-02-04T14:01:20.455Z 3 060-079 15 15 15 23 23 23 F F F
25242024-02-04T14:01:20.455Z 4 080-099 20 20 20 36 36 36 F F F
25252024-02-04T14:01:20.455Z 5 100-119 22 20 22 38 36 38 F F F <---
25262024-02-04T14:01:20.456Z 6 120-139 22 20 22 38 36 38 T F T <---
25272024-02-04T14:01:20.456Z 7 140-159 22 22 22 38 37 38 F F F <---
25282024-02-04T14:01:20.476Z 8 160-179 22 22 22 37 37 37 F F F
25292024-02-04T14:01:20.477Z 9 180-199 20 20 20 36 36 36 T F T <---
25302024-02-04T14:01:20.477Z 10 200-219 22 20 22 38 36 38 T T T <---
25312024-02-04T14:01:20.477Z 11 220-239 22 20 22 38 36 38 F T F <---
25322024-02-04T14:01:20.477Z 12 240-259 20 20 20 36 36 36 F F F
25332024-02-04T14:01:20.477Z 13 260-279 20 20 20 36 36 36 F F F
25342024-02-04T14:01:20.477Z 14 280-299 20 20 20 36 36 36 T F T <---
25352024-02-04T14:01:20.478Z 15 300-319 22 18 22 38 32 38 F T F <---
25362024-02-04T14:01:20.478Z 16 320-339 15 15 15 23 23 23 F F F
25372024-02-04T14:01:20.478Z 17 340-359 20 20 20 36 36 36 F F F
25382024-02-04T14:01:20.478Z 18 360-379 20 20 20 36 36 36 F F F
25392024-02-04T14:01:20.478Z 19 380-399 15 15 15 23 23 23 F F F
25402024-02-04T14:01:20.478Z 20 400-419 16 16 16 24 24 24 F F F
25412024-02-04T14:01:20.479Z 21 420-439 20 20 20 36 36 36 F F F
25422024-02-04T14:01:20.479Z 22 440-459 15 15 15 23 23 23 F F F
25432024-02-04T14:01:20.479Z 23 460-479 18 18 18 33 33 33 T F T <---
25442024-02-04T14:01:20.479Z 24 480-499 22 18 22 38 33 38 F F F <---
25452024-02-04T14:01:20.479Z 25 500-519 16 16 16 25 25 25 F F F
25462024-02-04T14:01:20.479Z 26 520-539 20 20 20 36 36 36 T F T <---
25472024-02-04T14:01:20.479Z 27 540-559 19 19 19 35 35 35 F F F
25482024-02-04T14:01:20.479Z 28 560-579 16 16 16 24 24 24 T F T <---
25492024-02-04T14:01:20.479Z 29 580-599 11 11 11 19 19 19 T F T <---
25502024-02-04T14:01:20.479ZMax gen: 22, Max flush: 38
25512024-02-04T14:01:20.479ZError: Difference in extent metadata found!
25522024-02-04T14:01:20.492ZA Difference in extent metadata is expected here
25532024-02-04T14:01:20.492ZOn loop 11
25542024-02-04T14:01:20.492Z
25552024-02-04T14:01:20.492ZVerifying data now
25562024-02-04T14:01:26.632ZNo change after verify
25572024-02-04T14:01:26.632ZLoop: 11 Downstairs dump after verify (and repair):
25582024-02-04T14:01:26.655ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25592024-02-04T14:01:26.655Z 0 000-019 2 2 2 3 3 3 F F F
25602024-02-04T14:01:26.655Z 1 020-039 19 19 19 35 35 35 F F F
25612024-02-04T14:01:26.655Z 2 040-059 15 15 15 23 23 23 F F F
25622024-02-04T14:01:26.655Z 3 060-079 15 15 15 23 23 23 F F F
25632024-02-04T14:01:26.655Z 4 080-099 20 20 20 36 36 36 F F F
25642024-02-04T14:01:26.655Z 5 100-119 22 22 22 38 38 38 F F F
25652024-02-04T14:01:26.655Z 6 120-139 23 23 23 39 39 39 F F F
25662024-02-04T14:01:26.659Z 7 140-159 22 22 22 38 38 38 F F F
25672024-02-04T14:01:26.659Z 8 160-179 22 22 22 37 37 37 F F F
25682024-02-04T14:01:26.659Z 9 180-199 23 23 23 39 39 39 F F F
25692024-02-04T14:01:26.659Z 10 200-219 23 23 23 39 39 39 F F F
25702024-02-04T14:01:26.659Z 11 220-239 22 22 22 38 38 38 F F F
25712024-02-04T14:01:26.659Z 12 240-259 20 20 20 36 36 36 F F F
25722024-02-04T14:01:26.659Z 13 260-279 20 20 20 36 36 36 F F F
25732024-02-04T14:01:26.659Z 14 280-299 23 23 23 39 39 39 F F F
25742024-02-04T14:01:26.659Z 15 300-319 22 22 22 38 38 38 F F F
25752024-02-04T14:01:26.659Z 16 320-339 15 15 15 23 23 23 F F F
25762024-02-04T14:01:26.659Z 17 340-359 20 20 20 36 36 36 F F F
25772024-02-04T14:01:26.659Z 18 360-379 20 20 20 36 36 36 F F F
25782024-02-04T14:01:26.659Z 19 380-399 15 15 15 23 23 23 F F F
25792024-02-04T14:01:26.660Z 20 400-419 16 16 16 24 24 24 F F F
25802024-02-04T14:01:26.660Z 21 420-439 20 20 20 36 36 36 F F F
25812024-02-04T14:01:26.660Z 22 440-459 15 15 15 23 23 23 F F F
25822024-02-04T14:01:26.660Z 23 460-479 23 23 23 39 39 39 F F F
25832024-02-04T14:01:26.660Z 24 480-499 22 22 22 38 38 38 F F F
25842024-02-04T14:01:26.660Z 25 500-519 16 16 16 25 25 25 F F F
25852024-02-04T14:01:26.660Z 26 520-539 23 23 23 39 39 39 F F F
25862024-02-04T14:01:26.660Z 27 540-559 19 19 19 35 35 35 F F F
25872024-02-04T14:01:26.660Z 28 560-579 23 23 23 39 39 39 F F F
25882024-02-04T14:01:26.660Z 29 580-599 23 23 23 39 39 39 F F F
25892024-02-04T14:01:26.660ZMax gen: 23, Max flush: 39
25902024-02-04T14:01:26.663Z
25912024-02-04T14:01:26.663ZBegin loop 12
25922024-02-04T14:01:26.663ZDownstairs to restart: 2
25932024-02-04T14:01:26.692Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.725453444Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225}
25942024-02-04T14:01:26.692Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.725912907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225}
25952024-02-04T14:01:26.692Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.725978043Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225}
25962024-02-04T14:01:26.693Z{"msg":"Crucible be326f4d-efac-47fc-9cc8-5deabf4ae89b has session id: 935d5407-5d6f-4c9b-92b8-16832448f395","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726008869Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
25972024-02-04T14:01:26.715Z{"msg":"Crucible runtime is spawned
25982024-02-04T14:01:26.715ZUpstairs opts: Upstairs UUID: be326f4d-efac-47fc-9cc8-5deabf4ae89b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726133039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
25992024-02-04T14:01:26.715Z{"msg":"Crucible stats registered with UUID: be326f4d-efac-47fc-9cc8-5deabf4ae89b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726239099Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26002024-02-04T14:01:26.716Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726279215Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225}
26012024-02-04T14:01:26.716Z{"msg":"be326f4d-efac-47fc-9cc8-5deabf4ae89b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726513695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26022024-02-04T14:01:26.716Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726605251Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26032024-02-04T14:01:26.716Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726665474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26042024-02-04T14:01:26.716Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726690543Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26052024-02-04T14:01:26.716Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726714849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26062024-02-04T14:01:26.716Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726803478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26072024-02-04T14:01:26.716Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726848522Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26082024-02-04T14:01:26.716Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.726981205Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26092024-02-04T14:01:26.716Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.727068009Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26102024-02-04T14:01:26.716Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.727137897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26112024-02-04T14:01:26.716Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.727185533Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26122024-02-04T14:01:26.716Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.727227239Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26132024-02-04T14:01:26.716Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:25.727305606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26142024-02-04T14:01:26.716Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:25.727386645Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26152024-02-04T14:01:26.716Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:25.727467642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26162024-02-04T14:01:26.716Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.727512742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26172024-02-04T14:01:26.716Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.727609925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26182024-02-04T14:01:26.716Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.727653523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26192024-02-04T14:01:26.716Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.728384667Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26202024-02-04T14:01:26.716Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.728455479Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26212024-02-04T14:01:26.717Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.728482395Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26222024-02-04T14:01:26.717Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.728609093Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26232024-02-04T14:01:26.717Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.729281567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26242024-02-04T14:01:26.717Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.729573553Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26252024-02-04T14:01:26.717Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.730115845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26262024-02-04T14:01:26.717Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.730181077Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26272024-02-04T14:01:26.717Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.73020691Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26282024-02-04T14:01:26.717Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.730409042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26292024-02-04T14:01:26.717Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.730481312Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26302024-02-04T14:01:26.717Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:25.730509397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26312024-02-04T14:01:27.695Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.728635Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26322024-02-04T14:01:27.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.728832092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26332024-02-04T14:01:27.700Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.729782137Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26342024-02-04T14:01:27.700Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.729851877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26352024-02-04T14:01:27.700Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.73054471Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26362024-02-04T14:01:27.700Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731337459Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26372024-02-04T14:01:27.700Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731393625Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26382024-02-04T14:01:27.700Z{"msg":"[0]R flush_numbers[0..12]: [3, 35, 23, 23, 36, 38, 39, 38, 37, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731420278Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26392024-02-04T14:01:27.700Z{"msg":"[0]R generation[0..12]: [2, 19, 15, 15, 20, 22, 23, 22, 22, 23, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731444123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26402024-02-04T14:01:27.700Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731475946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26412024-02-04T14:01:27.700Z{"msg":"[1]R flush_numbers[0..12]: [3, 35, 23, 23, 36, 38, 39, 38, 37, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.73153663Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26422024-02-04T14:01:27.700Z{"msg":"[1]R generation[0..12]: [2, 19, 15, 15, 20, 22, 23, 22, 22, 23, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731566671Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26432024-02-04T14:01:27.700ZWait for a query_work_queue command to finish before sending IO
26442024-02-04T14:01:27.701Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731632679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26452024-02-04T14:01:27.701Z{"msg":"[2]R flush_numbers[0..12]: [3, 35, 23, 23, 36, 38, 39, 38, 37, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731695573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26462024-02-04T14:01:27.701Z{"msg":"[2]R generation[0..12]: [2, 19, 15, 15, 20, 22, 23, 22, 22, 23, 23, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731728341Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26472024-02-04T14:01:27.701Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731769829Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26482024-02-04T14:01:27.701Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731836666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26492024-02-04T14:01:27.701Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731894012Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26502024-02-04T14:01:27.701Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.73192313Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26512024-02-04T14:01:27.701Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.731991637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26522024-02-04T14:01:27.701Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.732040962Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26532024-02-04T14:01:27.701Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.732125785Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26542024-02-04T14:01:27.701Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.732157383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26552024-02-04T14:01:27.701Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.732236859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26562024-02-04T14:01:27.701Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.73227432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26572024-02-04T14:01:27.701Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.732321742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26582024-02-04T14:01:27.701Z{"msg":"be326f4d-efac-47fc-9cc8-5deabf4ae89b is now active with session: 935d5407-5d6f-4c9b-92b8-16832448f395","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.732345492Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26592024-02-04T14:01:27.701ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26602024-02-04T14:01:27.701Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.732452136Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225,"session_id":"935d5407-5d6f-4c9b-92b8-16832448f395"}
26612024-02-04T14:01:27.701Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:26.732500906Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225}
26622024-02-04T14:01:27.705ZLoading write count information from file "/tmp/test_repair_verify.data"
26632024-02-04T14:01:27.705ZRun Repair workload
26642024-02-04T14:01:27.705Z01/30 Write block 283 len 40960 data: 2 2 2 2 3 4 4 4 4 4
26652024-02-04T14:01:27.705Z02/30 Write block 64 len 20480 data: 4 4 3 3 3
26662024-02-04T14:01:27.708Z03/30 Write block 504 len 24576 data: 4 4 5 5 4 5
26672024-02-04T14:01:27.708Z04/30 Read block 464 len 40960
26682024-02-04T14:01:27.711Z05/30 Write block 163 len 4096 data: 2
26692024-02-04T14:01:27.711Z06/30 Write block 290 len 12288 data: 5 5 5
26702024-02-04T14:01:27.711Z07/30 Read block 338 len 28672
26712024-02-04T14:01:27.714Z08/30 Write block 153 len 28672 data: 3 2 2 2 2 2 3
26722024-02-04T14:01:27.714Z09/30 Write block 31 len 40960 data: 4 3 3 3 4 4 4 2 3 2
26732024-02-04T14:01:27.714Z10/30 Flush
26742024-02-04T14:01:27.720Z11/30 Write block 282 len 36864 data: 3 3 3 3 3 4 5 5 6
26752024-02-04T14:01:27.723Z12/30 Read block 570 len 36864
26762024-02-04T14:01:27.727Z13/30 Read block 527 len 20480
26772024-02-04T14:01:27.727Z14/30 Write block 508 len 36864 data: 5 6 5 5 3 2 3 4 4
26782024-02-04T14:01:27.727Z15/30 Read block 563 len 4096
26792024-02-04T14:01:27.731Z16/30 Read block 524 len 36864
26802024-02-04T14:01:27.731Z17/30 Read block 268 len 40960
26812024-02-04T14:01:27.734Z18/30 Write block 472 len 24576 data: 3 3 4 3 3 3
26822024-02-04T14:01:27.734Z19/30 Flush
26832024-02-04T14:01:27.737Z20/30 Write block 140 len 4096 data: 3
26842024-02-04T14:01:27.737Z21/30 Read block 358 len 8192
26852024-02-04T14:01:27.740Z22/30 Read block 319 len 28672
26862024-02-04T14:01:27.740Z23/30 Flush
26872024-02-04T14:01:27.743Z24/30 Write block 411 len 4096 data: 4
26882024-02-04T14:01:27.743Z25/30 Write block 126 len 12288 data: 5 3 2
26892024-02-04T14:01:27.743Z26/30 Write block 206 len 20480 data: 3 3 3 2 2
26902024-02-04T14:01:27.749Z27/30 Write block 411 len 12288 data: 5 4 4
26912024-02-04T14:01:27.749Z28/30 Write block 298 len 40960 data: 3 3 2 3 5 5 5 4 3 3
26922024-02-04T14:01:27.753Z29/30 Write block 81 len 32768 data: 3 3 2 3 3 3 3 3
26932024-02-04T14:01:27.753Z30/30 Read block 361 len 36864
26942024-02-04T14:01:27.753Z----------------------------------------------------------------
26952024-02-04T14:01:27.753Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
26962024-02-04T14:01:27.753ZStates: Active Active Active
26972024-02-04T14:01:27.754ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26982024-02-04T14:01:27.754Z 1 Acked 1000 Write 10 Done Done Done false
26992024-02-04T14:01:27.754Z 2 Acked 1001 Write 5 Done Done Done false
27002024-02-04T14:01:27.754Z 3 Acked 1002 Write 6 Done Done Done false
27012024-02-04T14:01:27.754Z 4 Acked 1003 Read 10 Done Done Done false
27022024-02-04T14:01:27.754Z 5 Acked 1004 Write 1 Done Done Done false
27032024-02-04T14:01:27.754Z 6 Acked 1005 Write 3 Done Done Sent false
27042024-02-04T14:01:27.754Z 7 Acked 1006 Read 7 Done Done Sent false
27052024-02-04T14:01:27.754Z 8 Acked 1007 Write 7 Done Done Sent false
27062024-02-04T14:01:27.754Z 9 Acked 1008 Write 10 Done Done Sent false
27072024-02-04T14:01:27.754Z 10 Acked 1009 Flush 0 Done Done Sent false
27082024-02-04T14:01:27.754Z 11 Acked 1010 Write 9 Done Done Sent false
27092024-02-04T14:01:27.754Z 12 Acked 1011 Read 9 Done Done Sent false
27102024-02-04T14:01:27.754Z 13 Acked 1012 Read 5 Done Done Sent false
27112024-02-04T14:01:27.754Z 14 Acked 1013 Write 9 Done Done Sent false
27122024-02-04T14:01:27.754Z 15 Acked 1014 Read 1 Done Done Sent false
27132024-02-04T14:01:27.754Z 16 Acked 1015 Read 9 Done Done Sent false
27142024-02-04T14:01:27.754Z 17 Acked 1016 Read 10 Done Done Sent false
27152024-02-04T14:01:27.754Z 18 Acked 1017 Write 6 Done Done Sent false
27162024-02-04T14:01:27.754Z 19 Acked 1018 Flush 0 Done Done Sent false
27172024-02-04T14:01:27.754Z 20 Acked 1019 Write 1 Done Done Sent false
27182024-02-04T14:01:27.754Z 21 Acked 1020 Read 2 Done Done Sent false
27192024-02-04T14:01:27.754Z 22 Acked 1021 Read 7 Done Done Sent false
27202024-02-04T14:01:27.754Z 23 Acked 1022 Flush 0 Done Done Sent false
27212024-02-04T14:01:27.754Z 24 Acked 1023 Write 1 Done Done Sent false
27222024-02-04T14:01:27.754Z 25 Acked 1024 Write 3 Done Done Sent false
27232024-02-04T14:01:27.755Z 26 Acked 1025 Write 5 Done Done Sent false
27242024-02-04T14:01:27.755Z 27 Acked 1026 Write 3 Done Done Sent false
27252024-02-04T14:01:27.755Z 28 Acked 1027 Write 10 Done Done Sent false
27262024-02-04T14:01:27.755Z 29 Acked 1028 Write 8 Done Done Sent false
27272024-02-04T14:01:27.755Z 30 Acked 1029 Read 9 Sent Done Sent false
27282024-02-04T14:01:27.755Z STATES DS:0 DS:1 DS:2 TOTAL
27292024-02-04T14:01:27.755Z New 0 0 0 0
27302024-02-04T14:01:27.755Z Sent 1 0 25 26
27312024-02-04T14:01:27.755Z Done 29 30 5 64
27322024-02-04T14:01:27.755Z Skipped 0 0 0 0
27332024-02-04T14:01:27.755Z Error 0 0 0 0
27342024-02-04T14:01:27.755ZLast Flush: 1022 1022 0
27352024-02-04T14:01:27.755ZDownstairs last five completed:
27362024-02-04T14:01:27.755ZUpstairs last five completed: 30 29 28 27 26
27372024-02-04T14:01:27.755Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:26.784860449Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1225}
27382024-02-04T14:01:27.755ZWrote out file "/tmp/test_repair_verify.data"
27392024-02-04T14:01:27.758Z
27402024-02-04T14:01:27.766ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27412024-02-04T14:01:27.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27422024-02-04T14:01:27.792Z 0 000-019 2 2 2 3 3 3 F F F
27432024-02-04T14:01:27.792Z 1 020-039 24 24 19 40 40 35 F F F <---
27442024-02-04T14:01:27.792Z 2 040-059 24 24 15 40 40 23 F F F <---
27452024-02-04T14:01:27.792Z 3 060-079 24 24 15 40 40 23 F F T <---
27462024-02-04T14:01:27.792Z 4 080-099 20 20 20 36 36 36 T T F <---
27472024-02-04T14:01:27.793Z 5 100-119 22 22 22 38 38 38 F F F
27482024-02-04T14:01:27.793Z 6 120-139 23 23 23 39 39 39 T T F <---
27492024-02-04T14:01:27.793Z 7 140-159 24 24 22 42 42 38 F F F <---
27502024-02-04T14:01:27.793Z 8 160-179 24 24 22 40 40 37 F F T <---
27512024-02-04T14:01:27.796Z 9 180-199 23 23 23 39 39 39 F F F
27522024-02-04T14:01:27.796Z 10 200-219 23 23 23 39 39 39 T T F <---
27532024-02-04T14:01:27.796Z 11 220-239 22 22 22 38 38 38 F F F
27542024-02-04T14:01:27.796Z 12 240-259 20 20 20 36 36 36 F F F
27552024-02-04T14:01:27.796Z 13 260-279 20 20 20 36 36 36 F F F
27562024-02-04T14:01:27.796Z 14 280-299 24 24 23 41 41 39 T T T <---
27572024-02-04T14:01:27.796Z 15 300-319 22 22 22 38 38 38 T T F <---
27582024-02-04T14:01:27.796Z 16 320-339 15 15 15 23 23 23 F F F
27592024-02-04T14:01:27.796Z 17 340-359 20 20 20 36 36 36 F F F
27602024-02-04T14:01:27.796Z 18 360-379 20 20 20 36 36 36 F F F
27612024-02-04T14:01:27.796Z 19 380-399 15 15 15 23 23 23 F F F
27622024-02-04T14:01:27.796Z 20 400-419 16 16 16 24 24 24 T T F <---
27632024-02-04T14:01:27.797Z 21 420-439 20 20 20 36 36 36 F F F
27642024-02-04T14:01:27.797Z 22 440-459 15 15 15 23 23 23 F F F
27652024-02-04T14:01:27.797Z 23 460-479 24 24 23 41 41 39 F F F <---
27662024-02-04T14:01:27.797Z 24 480-499 22 22 22 38 38 38 F F F
27672024-02-04T14:01:27.797Z 25 500-519 24 24 16 41 41 25 F F T <---
27682024-02-04T14:01:27.797Z 26 520-539 23 23 23 39 39 39 F F F
27692024-02-04T14:01:27.797Z 27 540-559 19 19 19 35 35 35 F F F
27702024-02-04T14:01:27.797Z 28 560-579 23 23 23 39 39 39 F F F
27712024-02-04T14:01:27.797Z 29 580-599 23 23 23 39 39 39 F F F
27722024-02-04T14:01:27.797ZMax gen: 24, Max flush: 42
27732024-02-04T14:01:27.797ZError: Difference in extent metadata found!
27742024-02-04T14:01:27.800ZA Difference in extent metadata is expected here
27752024-02-04T14:01:27.800ZOn loop 12
27762024-02-04T14:01:27.800Z
27772024-02-04T14:01:27.803ZVerifying data now
27782024-02-04T14:01:33.991ZNo change after verify
27792024-02-04T14:01:33.991ZLoop: 12 Downstairs dump after verify (and repair):
27802024-02-04T14:01:34.015ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27812024-02-04T14:01:34.015Z 0 000-019 2 2 2 3 3 3 F F F
27822024-02-04T14:01:34.015Z 1 020-039 24 24 24 40 40 40 F F F
27832024-02-04T14:01:34.015Z 2 040-059 24 24 24 40 40 40 F F F
27842024-02-04T14:01:34.015Z 3 060-079 24 24 24 40 40 40 F F F
27852024-02-04T14:01:34.015Z 4 080-099 25 25 25 43 43 43 F F F
27862024-02-04T14:01:34.015Z 5 100-119 22 22 22 38 38 38 F F F
27872024-02-04T14:01:34.015Z 6 120-139 25 25 25 43 43 43 F F F
27882024-02-04T14:01:34.015Z 7 140-159 24 24 24 42 42 42 F F F
27892024-02-04T14:01:34.015Z 8 160-179 24 24 24 40 40 40 F F F
27902024-02-04T14:01:34.015Z 9 180-199 23 23 23 39 39 39 F F F
27912024-02-04T14:01:34.015Z 10 200-219 25 25 25 43 43 43 F F F
27922024-02-04T14:01:34.015Z 11 220-239 22 22 22 38 38 38 F F F
27932024-02-04T14:01:34.015Z 12 240-259 20 20 20 36 36 36 F F F
27942024-02-04T14:01:34.019Z 13 260-279 20 20 20 36 36 36 F F F
27952024-02-04T14:01:34.019Z 14 280-299 25 25 25 43 43 43 F F F
27962024-02-04T14:01:34.019Z 15 300-319 25 25 25 43 43 43 F F F
27972024-02-04T14:01:34.019Z 16 320-339 15 15 15 23 23 23 F F F
27982024-02-04T14:01:34.019Z 17 340-359 20 20 20 36 36 36 F F F
27992024-02-04T14:01:34.019Z 18 360-379 20 20 20 36 36 36 F F F
28002024-02-04T14:01:34.019Z 19 380-399 15 15 15 23 23 23 F F F
28012024-02-04T14:01:34.019Z 20 400-419 25 25 25 43 43 43 F F F
28022024-02-04T14:01:34.019Z 21 420-439 20 20 20 36 36 36 F F F
28032024-02-04T14:01:34.019Z 22 440-459 15 15 15 23 23 23 F F F
28042024-02-04T14:01:34.020Z 23 460-479 24 24 24 41 41 41 F F F
28052024-02-04T14:01:34.020Z 24 480-499 22 22 22 38 38 38 F F F
28062024-02-04T14:01:34.020Z 25 500-519 24 24 24 41 41 41 F F F
28072024-02-04T14:01:34.020Z 26 520-539 23 23 23 39 39 39 F F F
28082024-02-04T14:01:34.020Z 27 540-559 19 19 19 35 35 35 F F F
28092024-02-04T14:01:34.020Z 28 560-579 23 23 23 39 39 39 F F F
28102024-02-04T14:01:34.020Z 29 580-599 23 23 23 39 39 39 F F F
28112024-02-04T14:01:34.020ZMax gen: 25, Max flush: 43
28122024-02-04T14:01:34.023Z
28132024-02-04T14:01:34.023ZBegin loop 13
28142024-02-04T14:01:34.023ZDownstairs to restart: 2
28152024-02-04T14:01:34.051Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.084472077Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233}
28162024-02-04T14:01:34.051Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.084961104Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233}
28172024-02-04T14:01:34.051Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.084990244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233}
28182024-02-04T14:01:34.051Z{"msg":"Crucible 71fb4835-bbd9-4836-a29b-094d4e26d747 has session id: da962d44-a6e4-4670-a081-759351140f6b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085011959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28192024-02-04T14:01:34.051Z{"msg":"Upstairs opts: Upstairs UUID: 71fb4835-bbd9-4836-a29b-094d4e26d747, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085080788Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28202024-02-04T14:01:34.055Z{"msg":"Crucible stats registered with UUID: 71fb4835-bbd9-4836-a29b-094d4e26d747","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085124028Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28212024-02-04T14:01:34.055ZCrucible runtime is spawned
28222024-02-04T14:01:34.055Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085482323Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233}
28232024-02-04T14:01:34.055Z{"msg":"71fb4835-bbd9-4836-a29b-094d4e26d747 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085542696Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28242024-02-04T14:01:34.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085567499Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28252024-02-04T14:01:34.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085642386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28262024-02-04T14:01:34.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085764031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28272024-02-04T14:01:34.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.085967623Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28282024-02-04T14:01:34.055Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.086021097Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28292024-02-04T14:01:34.055Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.086066109Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28302024-02-04T14:01:34.056Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.086142147Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28312024-02-04T14:01:34.056Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.086249849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28322024-02-04T14:01:34.056Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.086326617Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28332024-02-04T14:01:34.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.086361534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28342024-02-04T14:01:34.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.086426944Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28352024-02-04T14:01:34.056Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.086477081Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28362024-02-04T14:01:34.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.087772432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28372024-02-04T14:01:34.056Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.087895998Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28382024-02-04T14:01:34.056Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.087967123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28392024-02-04T14:01:34.056Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.088006813Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28402024-02-04T14:01:34.061Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.088749923Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28412024-02-04T14:01:34.061Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.088969822Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28422024-02-04T14:01:34.061Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.089237953Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28432024-02-04T14:01:34.061Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.089296847Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28442024-02-04T14:01:34.061Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.089535937Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28452024-02-04T14:01:34.061Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.089595296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28462024-02-04T14:01:34.061Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.089620601Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28472024-02-04T14:01:34.061Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.08983323Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28482024-02-04T14:01:34.061Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.089917406Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28492024-02-04T14:01:34.061Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.089946797Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28502024-02-04T14:01:34.061Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.090210326Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28512024-02-04T14:01:34.061Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.090917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28522024-02-04T14:01:34.061Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.090986713Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28532024-02-04T14:01:34.061Z{"msg":"[0]R flush_numbers[0..12]: [3, 40, 40, 40, 43, 38, 43, 42, 40, 39, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091016001Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28542024-02-04T14:01:34.061Z{"msg":"[0]R generation[0..12]: [2, 24, 24, 24, 25, 22, 25, 24, 24, 23, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091058262Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28552024-02-04T14:01:34.061Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091083864Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28562024-02-04T14:01:34.061Z{"msg":"[1]R flush_numbers[0..12]: [3, 40, 40, 40, 43, 38, 43, 42, 40, 39, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091126756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28572024-02-04T14:01:34.061Z{"msg":"[1]R generation[0..12]: [2, 24, 24, 24, 25, 22, 25, 24, 24, 23, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091150852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28582024-02-04T14:01:34.061Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
28592024-02-04T14:01:34.062Z,"time":"2024-02-04T14:01:33.091174808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28602024-02-04T14:01:34.062Z{"msg":"[2]R flush_numbers[0..12]: [3, 40, 40, 40, 43, 38, 43, 42, 40, 39, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091283367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28612024-02-04T14:01:34.062Z{"msg":"[2]R generation[0..12]: [2, 24, 24, 24, 25, 22, 25, 24, 24, 23, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.09131173Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28622024-02-04T14:01:34.062Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091355338Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28632024-02-04T14:01:34.062Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091379119Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28642024-02-04T14:01:34.062Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091420089Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28652024-02-04T14:01:34.062Z{"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091442638Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28662024-02-04T14:01:34.062Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091464807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28672024-02-04T14:01:34.062Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091516155Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28682024-02-04T14:01:34.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091539737Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28692024-02-04T14:01:34.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091588128Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28702024-02-04T14:01:34.062Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091611599Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28712024-02-04T14:01:34.062Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091652805Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28722024-02-04T14:01:34.062Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091674739Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28732024-02-04T14:01:34.062Z{"msg":"71fb4835-bbd9-4836-a29b-094d4e26d747 is now active with session: da962d44-a6e4-4670-a081-759351140f6b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091696664Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28742024-02-04T14:01:34.062Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091735624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233,"session_id":"da962d44-a6e4-4670-a081-759351140f6b"}
28752024-02-04T14:01:34.062Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:33.091757248Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233}
28762024-02-04T14:01:34.062ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28772024-02-04T14:01:34.062ZLoading write count information from file "/tmp/test_repair_verify.data"
28782024-02-04T14:01:34.062ZRun Repair workload
28792024-02-04T14:01:34.062Z01/30 Write block 95 len 20480 data: 2 2 2 2 3
28802024-02-04T14:01:34.066Z02/30 Read block 359 len 36864
28812024-02-04T14:01:34.066Z03/30 Write block 290 len 4096 data: 7
28822024-02-04T14:01:34.066Z04/30 Flush
28832024-02-04T14:01:34.069Z05/30 Write block 261 len 20480 data: 4 5 6 5 4
28842024-02-04T14:01:34.069Z06/30 Read block 128 len 24576
28852024-02-04T14:01:34.074Z07/30 Read block 74 len 12288
28862024-02-04T14:01:34.074Z08/30 Write block 470 len 4096 data: 3
28872024-02-04T14:01:34.075Z09/30 Read block 461 len 40960
28882024-02-04T14:01:34.077Z10/30 Read block 274 len 24576
28892024-02-04T14:01:34.077Z11/30 Flush
28902024-02-04T14:01:34.080Z12/30 Write block 559 len 36864 data: 4 5 6 5 5 5 4 4 3
28912024-02-04T14:01:34.080Z13/30 Write block 346 len 32768 data: 3 3 2 2 2 2 2 3
28922024-02-04T14:01:34.083Z14/30 Read block 379 len 36864
28932024-02-04T14:01:34.083Z15/30 Read block 191 len 24576
28942024-02-04T14:01:34.088Z16/30 Read block 164 len 28672
28952024-02-04T14:01:34.088Z17/30 Read block 187 len 20480
28962024-02-04T14:01:34.094Z18/30 Flush
28972024-02-04T14:01:34.094Z19/30 Write block 380 len 12288 data: 5 3 2
28982024-02-04T14:01:34.094Z20/30 Write block 391 len 16384 data: 4 5 4 3
28992024-02-04T14:01:34.094Z21/30 Read block 81 len 4096
29002024-02-04T14:01:34.097Z22/30 Read block 101 len 32768
29012024-02-04T14:01:34.097Z23/30 Write block 583 len 12288 data: 4 4 4
29022024-02-04T14:01:34.097Z24/30 Read block 463 len 36864
29032024-02-04T14:01:34.100Z25/30 Write block 161 len 36864 data: 3 3 3 3 3 3 3 4 5
29042024-02-04T14:01:34.100Z26/30 Write block 250 len 12288 data: 6 5 5
29052024-02-04T14:01:34.100Z27/30 Read block 156 len 8192
29062024-02-04T14:01:34.103Z28/30 Write block 337 len 16384 data: 4 3 3 3
29072024-02-04T14:01:34.103Z29/30 Write block 108 len 4096 data: 4
29082024-02-04T14:01:34.103Z30/30 Write block 413 len 12288 data: 5 5 5
29092024-02-04T14:01:34.107Z----------------------------------------------------------------
29102024-02-04T14:01:34.107Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29112024-02-04T14:01:34.107ZStates: Active Active Active
29122024-02-04T14:01:34.107ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29132024-02-04T14:01:34.107Z 1 Acked 1000 Write 5 Done Done Sent false
29142024-02-04T14:01:34.107Z 2 Acked 1001 Read 9 Done Done Sent false
29152024-02-04T14:01:34.107Z 3 Acked 1002 Write 1 Done Done Sent false
29162024-02-04T14:01:34.107Z 4 Acked 1003 Flush 0 Done Done Sent false
29172024-02-04T14:01:34.107Z 5 Acked 1004 Write 5 Done Done Sent false
29182024-02-04T14:01:34.107Z 6 Acked 1005 Read 6 Done Done Sent false
29192024-02-04T14:01:34.107Z 7 Acked 1006 Read 3 Done Done Sent false
29202024-02-04T14:01:34.107Z 8 Acked 1007 Write 1 Done Done Sent false
29212024-02-04T14:01:34.107Z 9 Acked 1008 Read 10 Done Done Sent false
29222024-02-04T14:01:34.107Z 10 Acked 1009 Read 6 Done Done Sent false
29232024-02-04T14:01:34.107Z 11 Acked 1010 Flush 0 Done Done Sent false
29242024-02-04T14:01:34.107Z 12 Acked 1011 Write 9 Done Done Sent false
29252024-02-04T14:01:34.107Z 13 Acked 1012 Write 8 Done Done Sent false
29262024-02-04T14:01:34.107Z 14 Acked 1013 Read 9 Done Done Sent false
29272024-02-04T14:01:34.107Z 15 Acked 1014 Read 6 Done Done Sent false
29282024-02-04T14:01:34.107Z 16 Acked 1015 Read 7 Done Done Sent false
29292024-02-04T14:01:34.107Z 17 Acked 1016 Read 5 Done Done Sent false
29302024-02-04T14:01:34.108Z 18 Acked 1017 Flush 0 Done Done Sent false
29312024-02-04T14:01:34.108Z 19 Acked 1018 Write 3 Done Done Sent false
29322024-02-04T14:01:34.108Z 20 Acked 1019 Write 4 Done Done Sent false
29332024-02-04T14:01:34.108Z 21 Acked 1020 Read 1 Done Done Sent false
29342024-02-04T14:01:34.108Z 22 Acked 1021 Read 8 Done Done Sent false
29352024-02-04T14:01:34.108Z 23 Acked 1022 Write 3 Done Done Sent false
29362024-02-04T14:01:34.108Z 24 Acked 1023 Read 9 Done Done Sent false
29372024-02-04T14:01:34.108Z 25 Acked 1024 Write 9 Done Done Sent false
29382024-02-04T14:01:34.108Z 26 Acked 1025 Write 3 Done Done Sent false
29392024-02-04T14:01:34.108Z 27 Acked 1026 Read 2 Done Done Sent false
29402024-02-04T14:01:34.108Z 28 Acked 1027 Write 4 Done Done Sent false
29412024-02-04T14:01:34.108Z 29 Acked 1028 Write 1 Done Done Sent false
29422024-02-04T14:01:34.108Z 30 Acked 1029 Write 3 Sent Sent Sent false
29432024-02-04T14:01:34.108Z STATES DS:0 DS:1 DS:2 TOTAL
29442024-02-04T14:01:34.108Z New 0 0 0 0
29452024-02-04T14:01:34.108Z Sent 1 1 30 32
29462024-02-04T14:01:34.108Z Done 29 29 0 58
29472024-02-04T14:01:34.108Z Skipped 0 0 0 0
29482024-02-04T14:01:34.108Z Error 0 0 0 0
29492024-02-04T14:01:34.108ZLast Flush: 1017 1017 0
29502024-02-04T14:01:34.108ZDownstairs last five completed:
29512024-02-04T14:01:34.108ZUpstairs last five completed: 30 29 28 27 26
29522024-02-04T14:01:34.108Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:33.137611612Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1233}
29532024-02-04T14:01:34.108ZWrote out file "/tmp/test_repair_verify.data"
29542024-02-04T14:01:34.112Z
29552024-02-04T14:01:34.122ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29562024-02-04T14:01:34.148ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29572024-02-04T14:01:34.148Z 0 000-019 2 2 2 3 3 3 F F F
29582024-02-04T14:01:34.148Z 1 020-039 24 24 24 40 40 40 F F F
29592024-02-04T14:01:34.148Z 2 040-059 24 24 24 40 40 40 F F F
29602024-02-04T14:01:34.148Z 3 060-079 24 24 24 40 40 40 F F F
29612024-02-04T14:01:34.148Z 4 080-099 26 26 25 44 44 43 F F F <---
29622024-02-04T14:01:34.148Z 5 100-119 22 22 22 38 38 38 T T F <---
29632024-02-04T14:01:34.148Z 6 120-139 25 25 25 43 43 43 F F F
29642024-02-04T14:01:34.148Z 7 140-159 24 24 24 42 42 42 F F F
29652024-02-04T14:01:34.148Z 8 160-179 24 24 24 40 40 40 T T F <---
29662024-02-04T14:01:34.148Z 9 180-199 23 23 23 39 39 39 F F F
29672024-02-04T14:01:34.148Z 10 200-219 25 25 25 43 43 43 F F F
29682024-02-04T14:01:34.148Z 11 220-239 22 22 22 38 38 38 F F F
29692024-02-04T14:01:34.148Z 12 240-259 20 20 20 36 36 36 T T F <---
29702024-02-04T14:01:34.148Z 13 260-279 26 26 20 45 45 36 F F F <---
29712024-02-04T14:01:34.148Z 14 280-299 26 26 25 44 44 43 F F F <---
29722024-02-04T14:01:34.152Z 15 300-319 25 25 25 43 43 43 F F F
29732024-02-04T14:01:34.152Z 16 320-339 15 15 15 23 23 23 T T F <---
29742024-02-04T14:01:34.152Z 17 340-359 26 26 20 46 46 36 T T F <---
29752024-02-04T14:01:34.152Z 18 360-379 20 20 20 36 36 36 F F F
29762024-02-04T14:01:34.152Z 19 380-399 15 15 15 23 23 23 T T F <---
29772024-02-04T14:01:34.152Z 20 400-419 25 25 25 43 43 43 T T F <---
29782024-02-04T14:01:34.152Z 21 420-439 20 20 20 36 36 36 F F F
29792024-02-04T14:01:34.152Z 22 440-459 15 15 15 23 23 23 F F F
29802024-02-04T14:01:34.152Z 23 460-479 26 26 24 45 45 41 F F F <---
29812024-02-04T14:01:34.152Z 24 480-499 22 22 22 38 38 38 F F F
29822024-02-04T14:01:34.152Z 25 500-519 24 24 24 41 41 41 F F F
29832024-02-04T14:01:34.152Z 26 520-539 23 23 23 39 39 39 F F F
29842024-02-04T14:01:34.152Z 27 540-559 26 26 19 46 46 35 F F F <---
29852024-02-04T14:01:34.152Z 28 560-579 26 26 23 46 46 39 F F F <---
29862024-02-04T14:01:34.152Z 29 580-599 23 23 23 39 39 39 T T F <---
29872024-02-04T14:01:34.152ZMax gen: 26, Max flush: 46
29882024-02-04T14:01:34.152ZError: Difference in extent metadata found!
29892024-02-04T14:01:34.155ZA Difference in extent metadata is expected here
29902024-02-04T14:01:34.155ZOn loop 13
29912024-02-04T14:01:34.155Z
29922024-02-04T14:01:34.158ZVerifying data now
29932024-02-04T14:01:39.407ZNo change after verify
29942024-02-04T14:01:39.407ZLoop: 13 Downstairs dump after verify (and repair):
29952024-02-04T14:01:39.429ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29962024-02-04T14:01:39.429Z 0 000-019 2 2 2 3 3 3 F F F
29972024-02-04T14:01:39.429Z 1 020-039 24 24 24 40 40 40 F F F
29982024-02-04T14:01:39.429Z 2 040-059 24 24 24 40 40 40 F F F
29992024-02-04T14:01:39.430Z 3 060-079 24 24 24 40 40 40 F F F
30002024-02-04T14:01:39.430Z 4 080-099 26 26 26 44 44 44 F F F
30012024-02-04T14:01:39.430Z 5 100-119 27 27 27 47 47 47 F F F
30022024-02-04T14:01:39.430Z 6 120-139 25 25 25 43 43 43 F F F
30032024-02-04T14:01:39.430Z 7 140-159 24 24 24 42 42 42 F F F
30042024-02-04T14:01:39.430Z 8 160-179 27 27 27 47 47 47 F F F
30052024-02-04T14:01:39.430Z 9 180-199 23 23 23 39 39 39 F F F
30062024-02-04T14:01:39.430Z 10 200-219 25 25 25 43 43 43 F F F
30072024-02-04T14:01:39.430Z 11 220-239 22 22 22 38 38 38 F F F
30082024-02-04T14:01:39.430Z 12 240-259 27 27 27 47 47 47 F F F
30092024-02-04T14:01:39.430Z 13 260-279 26 26 26 45 45 45 F F F
30102024-02-04T14:01:39.430Z 14 280-299 26 26 26 44 44 44 F F F
30112024-02-04T14:01:39.431Z 15 300-319 25 25 25 43 43 43 F F F
30122024-02-04T14:01:39.434Z 16 320-339 27 27 27 47 47 47 F F F
30132024-02-04T14:01:39.434Z 17 340-359 27 27 27 47 47 47 F F F
30142024-02-04T14:01:39.434Z 18 360-379 20 20 20 36 36 36 F F F
30152024-02-04T14:01:39.434Z 19 380-399 27 27 27 47 47 47 F F F
30162024-02-04T14:01:39.435Z 20 400-419 27 27 27 47 47 47 F F F
30172024-02-04T14:01:39.435Z 21 420-439 20 20 20 36 36 36 F F F
30182024-02-04T14:01:39.435Z 22 440-459 15 15 15 23 23 23 F F F
30192024-02-04T14:01:39.435Z 23 460-479 26 26 26 45 45 45 F F F
30202024-02-04T14:01:39.435Z 24 480-499 22 22 22 38 38 38 F F F
30212024-02-04T14:01:39.435Z 25 500-519 24 24 24 41 41 41 F F F
30222024-02-04T14:01:39.435Z 26 520-539 23 23 23 39 39 39 F F F
30232024-02-04T14:01:39.435Z 27 540-559 26 26 26 46 46 46 F F F
30242024-02-04T14:01:39.435Z 28 560-579 26 26 26 46 46 46 F F F
30252024-02-04T14:01:39.435Z 29 580-599 27 27 27 47 47 47 F F F
30262024-02-04T14:01:39.435ZMax gen: 27, Max flush: 47
30272024-02-04T14:01:39.440Z
30282024-02-04T14:01:39.440ZBegin loop 14
30292024-02-04T14:01:39.440ZDownstairs to restart: 2
30302024-02-04T14:01:39.466Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.499259538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241}
30312024-02-04T14:01:39.466Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.499715251Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241}
30322024-02-04T14:01:39.471Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.499789746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241}
30332024-02-04T14:01:39.471Z{"msg":"Crucible 3538b643-0ad4-462f-b25f-f2316e047592 has session id: 9314d7dd-f976-40cd-ac02-015cd0f69b77","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.499921984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30342024-02-04T14:01:39.471Z{"msg":"Upstairs opts: Upstairs UUID: 3538b643-0ad4-462f-b25f-f2316e047592, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
30352024-02-04T14:01:39.471Z,"time":"2024-02-04T14:01:38.500088075Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30362024-02-04T14:01:39.471Z{"msg":"Crucible stats registered with UUID: 3538b643-0ad4-462f-b25f-f2316e047592","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500236967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30372024-02-04T14:01:39.471Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500285221Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241}
30382024-02-04T14:01:39.471Z{"msg":"3538b643-0ad4-462f-b25f-f2316e047592 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.50040162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30392024-02-04T14:01:39.471Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500468285Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30402024-02-04T14:01:39.472Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500559175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30412024-02-04T14:01:39.472Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.50064778Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30422024-02-04T14:01:39.472Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500673914Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30432024-02-04T14:01:39.472Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500706102Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30442024-02-04T14:01:39.472Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500747086Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30452024-02-04T14:01:39.472Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500814732Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30462024-02-04T14:01:39.472Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.500860807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30472024-02-04T14:01:39.472Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.501016211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30482024-02-04T14:01:39.472Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.50108662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30492024-02-04T14:01:39.472Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.501114736Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30502024-02-04T14:01:39.472Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.501138483Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30512024-02-04T14:01:39.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.502427453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30522024-02-04T14:01:39.472Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.502488212Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30532024-02-04T14:01:39.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.502569984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30542024-02-04T14:01:39.472Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.502621847Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30552024-02-04T14:01:39.473Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.503377322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30562024-02-04T14:01:39.473Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.503571774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30572024-02-04T14:01:39.473Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.503713053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30582024-02-04T14:01:39.473Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.503799832Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30592024-02-04T14:01:39.477Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.504265514Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30602024-02-04T14:01:39.477Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.504353451Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30612024-02-04T14:01:39.477Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.504452692Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30622024-02-04T14:01:39.478Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.504617795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30632024-02-04T14:01:39.478Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.504690223Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30642024-02-04T14:01:39.478Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.504715882Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30652024-02-04T14:01:39.478Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.504739731Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30662024-02-04T14:01:39.478Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505473167Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30672024-02-04T14:01:39.478Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505559024Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30682024-02-04T14:01:39.478Z{"msg":"[0]R flush_numbers[0..12]: [3, 40, 40, 40, 44, 47, 43, 42, 47, 39, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505634756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30692024-02-04T14:01:39.478Z{"msg":"[0]R generation[0..12]: [2, 24, 24, 24, 26, 27, 25, 24, 27, 23, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505686733Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30702024-02-04T14:01:39.478Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505713053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30712024-02-04T14:01:39.478ZWait for a query_work_queue command to finish before sending IO
30722024-02-04T14:01:39.478Z{"msg":"[1]R flush_numbers[0..12]: [3, 40, 40, 40, 44, 47, 43, 42, 47, 39, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505742236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30732024-02-04T14:01:39.478Z{"msg":"[1]R generation[0..12]: [2, 24, 24, 24, 26, 27, 25, 24, 27, 23, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505765536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30742024-02-04T14:01:39.478Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505809224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30752024-02-04T14:01:39.478Z{"msg":"[2]R flush_numbers[0..12]: [3, 40, 40, 40, 44, 47, 43, 42, 47, 39, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505833972Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30762024-02-04T14:01:39.478Z{"msg":"[2]R generation[0..12]: [2, 24, 24, 24, 26, 27, 25, 24, 27, 23, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505857341Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30772024-02-04T14:01:39.478Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505881155Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30782024-02-04T14:01:39.478Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505905706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30792024-02-04T14:01:39.478Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505928863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30802024-02-04T14:01:39.478Z{"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505950577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30812024-02-04T14:01:39.479Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.505993384Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30822024-02-04T14:01:39.479Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506017788Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30832024-02-04T14:01:39.479Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506038335Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30842024-02-04T14:01:39.479Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506060224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30852024-02-04T14:01:39.479Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506082025Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30862024-02-04T14:01:39.479Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506122852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30872024-02-04T14:01:39.479Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506145194Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30882024-02-04T14:01:39.479Z{"msg":"3538b643-0ad4-462f-b25f-f2316e047592 is now active with session: 9314d7dd-f976-40cd-ac02-015cd0f69b77","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506191695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30892024-02-04T14:01:39.479Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506212938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241,"session_id":"9314d7dd-f976-40cd-ac02-015cd0f69b77"}
30902024-02-04T14:01:39.479Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:38.506261201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241}
30912024-02-04T14:01:39.479ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30922024-02-04T14:01:39.479ZLoading write count information from file "/tmp/test_repair_verify.data"
30932024-02-04T14:01:39.479ZRun Repair workload
30942024-02-04T14:01:39.479Z01/30 Write block 191 len 32768 data: 3 4 3 2 2 3 3 5
30952024-02-04T14:01:39.479Z02/30 Read block 499 len 16384
30962024-02-04T14:01:39.493Z03/30 Read block 510 len 40960
30972024-02-04T14:01:39.493Z04/30 Read block 570 len 16384
30982024-02-04T14:01:39.493Z05/30 Read block 162 len 40960
30992024-02-04T14:01:39.496Z06/30 Write block 491 len 36864 data: 4 4 4 4 4 5 4 5 5
31002024-02-04T14:01:39.496Z07/30 Read block 110 len 20480
31012024-02-04T14:01:39.496Z08/30 Read block 48 len 24576
31022024-02-04T14:01:39.496Z09/30 Write block 277 len 12288 data: 3 4 4
31032024-02-04T14:01:39.496Z10/30 Read block 249 len 40960
31042024-02-04T14:01:39.496Z11/30 Read block 524 len 36864
31052024-02-04T14:01:39.496Z12/30 Read block 88 len 16384
31062024-02-04T14:01:39.497Z13/30 Read block 180 len 24576
31072024-02-04T14:01:39.500Z14/30 Write block 550 len 40960 data: 3 3 3 3 3 3 5 5 5 5
31082024-02-04T14:01:39.500Z15/30 Read block 99 len 12288
31092024-02-04T14:01:39.500Z16/30 Read block 328 len 32768
31102024-02-04T14:01:39.503Z17/30 Read block 370 len 8192
31112024-02-04T14:01:39.503Z18/30 Write block 425 len 32768 data: 2 2 2 2 2 3 3 3
31122024-02-04T14:01:39.503Z19/30 Read block 527 len 28672
31132024-02-04T14:01:39.507Z20/30 Read block 245 len 36864
31142024-02-04T14:01:39.510Z21/30 Write block 489 len 20480 data: 5 5 5 5 5
31152024-02-04T14:01:39.510Z22/30 Write block 572 len 40960 data: 2 2 2 2 3 3 3 3 3 4
31162024-02-04T14:01:39.513Z23/30 Write block 361 len 20480 data: 2 2 2 3 4
31172024-02-04T14:01:39.513Z24/30 Read block 273 len 20480
31182024-02-04T14:01:39.513Z25/30 Write block 359 len 28672 data: 2 2 3 3 3 4 5
31192024-02-04T14:01:39.516Z26/30 Read block 551 len 20480
31202024-02-04T14:01:39.516Z27/30 Write block 271 len 24576 data: 6 3 3 3 3 3
31212024-02-04T14:01:39.520Z28/30 Write block 183 len 16384 data: 5 5 5 4
31222024-02-04T14:01:39.520Z29/30 Write block 42 len 20480 data: 2 2 2 2 2
31232024-02-04T14:01:39.520Z30/30 Write block 466 len 8192 data: 2 2
31242024-02-04T14:01:39.520Z----------------------------------------------------------------
31252024-02-04T14:01:39.521Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31262024-02-04T14:01:39.521ZStates: Active Active Active
31272024-02-04T14:01:39.521ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31282024-02-04T14:01:39.521Z 1 Acked 1000 Write 8 Done Done Done false
31292024-02-04T14:01:39.521Z 2 Acked 1001 Read 4 Done Done Done false
31302024-02-04T14:01:39.521Z 3 Acked 1002 Read 10 Done Done Sent false
31312024-02-04T14:01:39.521Z 4 Acked 1003 Read 4 Done Done Sent false
31322024-02-04T14:01:39.521Z 5 Acked 1004 Read 10 Done Done Sent false
31332024-02-04T14:01:39.521Z 6 Acked 1005 Write 9 Done Done Sent false
31342024-02-04T14:01:39.521Z 7 Acked 1006 Read 5 Done Done Sent false
31352024-02-04T14:01:39.521Z 8 Acked 1007 Read 6 Done Done Sent false
31362024-02-04T14:01:39.521Z 9 Acked 1008 Write 3 Done Done Sent false
31372024-02-04T14:01:39.521Z 10 Acked 1009 Read 10 Done Done Sent false
31382024-02-04T14:01:39.521Z 11 Acked 1010 Read 9 Done Done Sent false
31392024-02-04T14:01:39.521Z 12 Acked 1011 Read 4 Done Done Sent false
31402024-02-04T14:01:39.522Z 13 Acked 1012 Read 6 Done Done Sent false
31412024-02-04T14:01:39.522Z 14 Acked 1013 Write 10 Done Done Sent false
31422024-02-04T14:01:39.522Z 15 Acked 1014 Read 3 Done Done Sent false
31432024-02-04T14:01:39.522Z 16 Acked 1015 Read 8 Done Done Sent false
31442024-02-04T14:01:39.522Z 17 Acked 1016 Read 2 Done Done Sent false
31452024-02-04T14:01:39.522Z 18 Acked 1017 Write 8 Done Done Sent false
31462024-02-04T14:01:39.522Z 19 Acked 1018 Read 7 Done Done Sent false
31472024-02-04T14:01:39.522Z 20 Acked 1019 Read 9 Done Done Sent false
31482024-02-04T14:01:39.522Z 21 Acked 1020 Write 5 Done Done Sent false
31492024-02-04T14:01:39.522Z 22 Acked 1021 Write 10 Done Done Sent false
31502024-02-04T14:01:39.522Z 23 Acked 1022 Write 5 Done Done Sent false
31512024-02-04T14:01:39.522Z 24 Acked 1023 Read 5 Done Done Sent false
31522024-02-04T14:01:39.522Z 25 Acked 1024 Write 7 Done Done Sent false
31532024-02-04T14:01:39.522Z 26 Acked 1025 Read 5 Done Done Sent false
31542024-02-04T14:01:39.522Z 27 Acked 1026 Write 6 Done Done Sent false
31552024-02-04T14:01:39.522Z 28 Acked 1027 Write 4 Done Done Sent false
31562024-02-04T14:01:39.523Z 29 Acked 1028 Write 5 Sent Sent Sent false
31572024-02-04T14:01:39.523Z 30 Acked 1029 Write 2 Sent Sent Sent false
31582024-02-04T14:01:39.523Z STATES DS:0 DS:1 DS:2 TOTAL
31592024-02-04T14:01:39.523Z New 0 0 0 0
31602024-02-04T14:01:39.523Z Sent 2 2 28 32
31612024-02-04T14:01:39.523Z Done 28 28 2 58
31622024-02-04T14:01:39.523Z Skipped 0 0 0 0
31632024-02-04T14:01:39.523Z Error 0 0 0 0
31642024-02-04T14:01:39.523ZLast Flush: 0 0 0
31652024-02-04T14:01:39.523ZDownstairs last five completed:
31662024-02-04T14:01:39.523ZUpstairs last five completed: 30 29 28 27 26
31672024-02-04T14:01:39.527Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:38.554183305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1241}
31682024-02-04T14:01:39.527ZWrote out file "/tmp/test_repair_verify.data"
31692024-02-04T14:01:39.530Z
31702024-02-04T14:01:39.535ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31712024-02-04T14:01:39.566ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31722024-02-04T14:01:39.566Z 0 000-019 2 2 2 3 3 3 F F F
31732024-02-04T14:01:39.567Z 1 020-039 24 24 24 40 40 40 F F F
31742024-02-04T14:01:39.567Z 2 040-059 24 24 24 40 40 40 T T F <---
31752024-02-04T14:01:39.567Z 3 060-079 24 24 24 40 40 40 F F F
31762024-02-04T14:01:39.567Z 4 080-099 26 26 26 44 44 44 F F F
31772024-02-04T14:01:39.567Z 5 100-119 27 27 27 47 47 47 F F F
31782024-02-04T14:01:39.567Z 6 120-139 25 25 25 43 43 43 F F F
31792024-02-04T14:01:39.567Z 7 140-159 24 24 24 42 42 42 F F F
31802024-02-04T14:01:39.567Z 8 160-179 27 27 27 47 47 47 F F F
31812024-02-04T14:01:39.567Z 9 180-199 23 23 23 39 39 39 T T T
31822024-02-04T14:01:39.567Z 10 200-219 25 25 25 43 43 43 F F F
31832024-02-04T14:01:39.567Z 11 220-239 22 22 22 38 38 38 F F F
31842024-02-04T14:01:39.567Z 12 240-259 27 27 27 47 47 47 F F F
31852024-02-04T14:01:39.567Z 13 260-279 26 26 26 45 45 45 T T F <---
31862024-02-04T14:01:39.568Z 14 280-299 26 26 26 44 44 44 F F F
31872024-02-04T14:01:39.568Z 15 300-319 25 25 25 43 43 43 F F F
31882024-02-04T14:01:39.574Z 16 320-339 27 27 27 47 47 47 F F F
31892024-02-04T14:01:39.575Z 17 340-359 27 27 27 47 47 47 T T F <---
31902024-02-04T14:01:39.575Z 18 360-379 20 20 20 36 36 36 T T F <---
31912024-02-04T14:01:39.575Z 19 380-399 27 27 27 47 47 47 F F F
31922024-02-04T14:01:39.575Z 20 400-419 27 27 27 47 47 47 F F F
31932024-02-04T14:01:39.575Z 21 420-439 20 20 20 36 36 36 T T F <---
31942024-02-04T14:01:39.575Z 22 440-459 15 15 15 23 23 23 F F F
31952024-02-04T14:01:39.575Z 23 460-479 26 26 26 45 45 45 T T F <---
31962024-02-04T14:01:39.575Z 24 480-499 22 22 22 38 38 38 T T F <---
31972024-02-04T14:01:39.575Z 25 500-519 24 24 24 41 41 41 F F F
31982024-02-04T14:01:39.575Z 26 520-539 23 23 23 39 39 39 F F F
31992024-02-04T14:01:39.575Z 27 540-559 26 26 26 46 46 46 T T F <---
32002024-02-04T14:01:39.575Z 28 560-579 26 26 26 46 46 46 T T F <---
32012024-02-04T14:01:39.575Z 29 580-599 27 27 27 47 47 47 T T F <---
32022024-02-04T14:01:39.576ZMax gen: 27, Max flush: 47
32032024-02-04T14:01:39.576ZError: Difference in extent metadata found!
32042024-02-04T14:01:39.576ZA Difference in extent metadata is expected here
32052024-02-04T14:01:39.576ZOn loop 14
32062024-02-04T14:01:39.576Z
32072024-02-04T14:01:39.579ZVerifying data now
32082024-02-04T14:01:44.614ZNo change after verify
32092024-02-04T14:01:44.614ZLoop: 14 Downstairs dump after verify (and repair):
32102024-02-04T14:01:44.637ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32112024-02-04T14:01:44.637Z 0 000-019 2 2 2 3 3 3 F F F
32122024-02-04T14:01:44.637Z 1 020-039 24 24 24 40 40 40 F F F
32132024-02-04T14:01:44.637Z 2 040-059 28 28 28 48 48 48 F F F
32142024-02-04T14:01:44.637Z 3 060-079 24 24 24 40 40 40 F F F
32152024-02-04T14:01:44.638Z 4 080-099 26 26 26 44 44 44 F F F
32162024-02-04T14:01:44.638Z 5 100-119 27 27 27 47 47 47 F F F
32172024-02-04T14:01:44.638Z 6 120-139 25 25 25 43 43 43 F F F
32182024-02-04T14:01:44.638Z 7 140-159 24 24 24 42 42 42 F F F
32192024-02-04T14:01:44.638Z 8 160-179 27 27 27 47 47 47 F F F
32202024-02-04T14:01:44.638Z 9 180-199 28 28 28 48 48 48 F F F
32212024-02-04T14:01:44.641Z 10 200-219 25 25 25 43 43 43 F F F
32222024-02-04T14:01:44.641Z 11 220-239 22 22 22 38 38 38 F F F
32232024-02-04T14:01:44.641Z 12 240-259 27 27 27 47 47 47 F F F
32242024-02-04T14:01:44.641Z 13 260-279 28 28 28 48 48 48 F F F
32252024-02-04T14:01:44.641Z 14 280-299 26 26 26 44 44 44 F F F
32262024-02-04T14:01:44.641Z 15 300-319 25 25 25 43 43 43 F F F
32272024-02-04T14:01:44.641Z 16 320-339 27 27 27 47 47 47 F F F
32282024-02-04T14:01:44.641Z 17 340-359 28 28 28 48 48 48 F F F
32292024-02-04T14:01:44.641Z 18 360-379 28 28 28 48 48 48 F F F
32302024-02-04T14:01:44.641Z 19 380-399 27 27 27 47 47 47 F F F
32312024-02-04T14:01:44.641Z 20 400-419 27 27 27 47 47 47 F F F
32322024-02-04T14:01:44.641Z 21 420-439 28 28 28 48 48 48 F F F
32332024-02-04T14:01:44.642Z 22 440-459 15 15 15 23 23 23 F F F
32342024-02-04T14:01:44.642Z 23 460-479 28 28 28 48 48 48 F F F
32352024-02-04T14:01:44.642Z 24 480-499 28 28 28 48 48 48 F F F
32362024-02-04T14:01:44.642Z 25 500-519 24 24 24 41 41 41 F F F
32372024-02-04T14:01:44.642Z 26 520-539 23 23 23 39 39 39 F F F
32382024-02-04T14:01:44.642Z 27 540-559 28 28 28 48 48 48 F F F
32392024-02-04T14:01:44.642Z 28 560-579 28 28 28 48 48 48 F F F
32402024-02-04T14:01:44.642Z 29 580-599 28 28 28 48 48 48 F F F
32412024-02-04T14:01:44.642ZMax gen: 28, Max flush: 48
32422024-02-04T14:01:44.645Z
32432024-02-04T14:01:44.645ZBegin loop 15
32442024-02-04T14:01:44.645ZDownstairs to restart: 1
32452024-02-04T14:01:44.677Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.710205292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249}
32462024-02-04T14:01:44.677Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.710666498Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249}
32472024-02-04T14:01:44.677Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.710696311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249}
32482024-02-04T14:01:44.681Z{"msg":"Crucible 804a2e12-4da9-4098-9141-3dc697072fd6 has session id: e8dbac46-6fa8-4b50-9c7c-7d4875b4e651","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.710733951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32492024-02-04T14:01:44.681Z{"msg":"Upstairs opts: Upstairs UUID: 804a2e12-4da9-4098-9141-3dc697072fd6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
32502024-02-04T14:01:44.681Z,"time":"2024-02-04T14:01:43.710813051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32512024-02-04T14:01:44.681Z{"msg":"Crucible stats registered with UUID: 804a2e12-4da9-4098-9141-3dc697072fd6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.710989156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32522024-02-04T14:01:44.681Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711139607Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249}
32532024-02-04T14:01:44.681Z{"msg":"804a2e12-4da9-4098-9141-3dc697072fd6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711325335Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32542024-02-04T14:01:44.681Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711388432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32552024-02-04T14:01:44.681Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711414874Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32562024-02-04T14:01:44.681Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711437564Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32572024-02-04T14:01:44.681Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711491569Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32582024-02-04T14:01:44.681Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711517364Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32592024-02-04T14:01:44.681Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711540214Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32602024-02-04T14:01:44.681Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711567174Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32612024-02-04T14:01:44.681Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711608552Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32622024-02-04T14:01:44.682Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711737969Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32632024-02-04T14:01:44.682Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711869975Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32642024-02-04T14:01:44.682Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711922631Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32652024-02-04T14:01:44.682Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.711973351Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32662024-02-04T14:01:44.682Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.713161246Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32672024-02-04T14:01:44.682Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.713231484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32682024-02-04T14:01:44.682Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.713257559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32692024-02-04T14:01:44.682Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.713285334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32702024-02-04T14:01:44.682Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.714205935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32712024-02-04T14:01:44.682Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.714366683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32722024-02-04T14:01:44.686Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.715133782Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32732024-02-04T14:01:44.686Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.715193038Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32742024-02-04T14:01:44.686Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.715218167Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32752024-02-04T14:01:44.686Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.715238721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32762024-02-04T14:01:44.686Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.715298676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32772024-02-04T14:01:44.686Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.715346086Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32782024-02-04T14:01:44.686Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.71536807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32792024-02-04T14:01:44.686Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.715399927Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32802024-02-04T14:01:44.686Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.716135583Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32812024-02-04T14:01:44.686Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.716824251Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32822024-02-04T14:01:44.686Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.716876877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32832024-02-04T14:01:44.687Z{"msg":"[0]R flush_numbers[0..12]: [3, 40, 48, 40, 44, 47, 43, 42, 47, 48, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.716902495Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32842024-02-04T14:01:44.687Z{"msg":"[0]R generation[0..12]: [2, 24, 28, 24, 26, 27, 25, 24, 27, 28, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.716926334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32852024-02-04T14:01:44.687Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.716969428Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32862024-02-04T14:01:44.687Z{"msg":"[1]R flush_numbers[0..12]: [3, 40, 48, 40, 44, 47, 43, 42, 47, 48, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.716994529Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32872024-02-04T14:01:44.687Z{"msg":"[1]R generation[0..12]: [2, 24, 28, 24, 26, 27, 25, 24, 27, 28, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717038016Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32882024-02-04T14:01:44.687Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717063438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32892024-02-04T14:01:44.687Z{"msg":"[2]R flush_numbers[0..12]: [3, 40, 48, 40, 44, 47, 43, 42, 47, 48, 43, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717112435Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32902024-02-04T14:01:44.687Z{"msg":"[2]R generation[0..12]: [2, 24, 28, 24, 26, 27, 25, 24, 27, 28, 25, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717136732Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32912024-02-04T14:01:44.687Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717187629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32922024-02-04T14:01:44.687Z{"msg":"Max found gen is 29","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
32932024-02-04T14:01:44.687Z,"time":"2024-02-04T14:01:43.717211049Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32942024-02-04T14:01:44.687Z{"msg":"Generation requested: 30 >= found:29","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717234573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32952024-02-04T14:01:44.687Z{"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717255867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32962024-02-04T14:01:44.687Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717277647Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32972024-02-04T14:01:44.687Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717299771Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32982024-02-04T14:01:44.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.71731978Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
32992024-02-04T14:01:44.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717367136Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
33002024-02-04T14:01:44.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717391086Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
33012024-02-04T14:01:44.687Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.71744683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
33022024-02-04T14:01:44.687Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.71747076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
33032024-02-04T14:01:44.687Z{"msg":"804a2e12-4da9-4098-9141-3dc697072fd6 is now active with session: e8dbac46-6fa8-4b50-9c7c-7d4875b4e651","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717512227Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
33042024-02-04T14:01:44.687Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717533613Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249,"session_id":"e8dbac46-6fa8-4b50-9c7c-7d4875b4e651"}
33052024-02-04T14:01:44.687Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:43.717576741Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249}
33062024-02-04T14:01:44.687ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33072024-02-04T14:01:44.688ZLoading write count information from file "/tmp/test_repair_verify.data"
33082024-02-04T14:01:44.688ZRun Repair workload
33092024-02-04T14:01:44.691Z01/30 Write block 358 len 24576 data: 2 3 3 4 4 4
33102024-02-04T14:01:44.691Z02/30 Flush
33112024-02-04T14:01:44.694Z03/30 Write block 534 len 40960 data: 4 4 4 2 2 2 3 4 3 4
33122024-02-04T14:01:44.694Z04/30 Write block 569 len 28672 data: 3 2 2 3 3 3 3
33132024-02-04T14:01:44.697Z05/30 Write block 416 len 8192 data: 3 3
33142024-02-04T14:01:44.697Z06/30 Read block 25 len 28672
33152024-02-04T14:01:44.700Z07/30 Read block 458 len 28672
33162024-02-04T14:01:44.700Z08/30 Write block 579 len 8192 data: 4 4
33172024-02-04T14:01:44.700Z09/30 Read block 360 len 40960
33182024-02-04T14:01:44.703Z10/30 Flush
33192024-02-04T14:01:44.706Z11/30 Write block 351 len 24576 data: 3 3 4 3 3 3
33202024-02-04T14:01:44.709Z12/30 Write block 175 len 32768 data: 6 5 5 5 6 6 6 5
33212024-02-04T14:01:44.709Z13/30 Read block 539 len 4096
33222024-02-04T14:01:44.709Z14/30 Read block 80 len 24576
33232024-02-04T14:01:44.712Z15/30 Write block 458 len 36864 data: 4 3 3 4 3 3 3 3 3
33242024-02-04T14:01:44.712Z16/30 Write block 240 len 4096 data: 3
33252024-02-04T14:01:44.715Z17/30 Read block 539 len 12288
33262024-02-04T14:01:44.715Z18/30 Write block 528 len 12288 data: 2 2 2
33272024-02-04T14:01:44.717Z19/30 Read block 140 len 20480
33282024-02-04T14:01:44.717Z20/30 Flush
33292024-02-04T14:01:44.721Z21/30 Write block 189 len 4096 data: 3
33302024-02-04T14:01:44.723Z22/30 Write block 231 len 16384 data: 2 2 2 2
33312024-02-04T14:01:44.724Z23/30 Flush
33322024-02-04T14:01:44.729Z24/30 Write block 466 len 32768 data: 4 3 2 3 4 3 4 4
33332024-02-04T14:01:44.730Z25/30 Read block 187 len 16384
33342024-02-04T14:01:44.733Z26/30 Read block 361 len 16384
33352024-02-04T14:01:44.733Z27/30 Read block 309 len 32768
33362024-02-04T14:01:44.733Z28/30 Write block 77 len 36864 data: 3 3 3 3 4 4 3 4 4
33372024-02-04T14:01:44.736Z29/30 Read block 99 len 24576
33382024-02-04T14:01:44.736Z30/30 Read block 205 len 32768
33392024-02-04T14:01:44.739Z----------------------------------------------------------------
33402024-02-04T14:01:44.739Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:28
33412024-02-04T14:01:44.739ZStates: Active Active Active
33422024-02-04T14:01:44.739ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33432024-02-04T14:01:44.739Z 3 Acked 1002 Write 10 Done Done Done false
33442024-02-04T14:01:44.739Z 4 Acked 1003 Write 7 Done Done Done false
33452024-02-04T14:01:44.739Z 5 Acked 1004 Write 2 Done Done Done false
33462024-02-04T14:01:44.739Z 6 Acked 1005 Read 7 Done Sent Done false
33472024-02-04T14:01:44.739Z 7 Acked 1006 Read 7 Done Sent Done false
33482024-02-04T14:01:44.739Z 8 Acked 1007 Write 2 Done Sent Done false
33492024-02-04T14:01:44.739Z 9 Acked 1008 Read 10 Done Sent Done false
33502024-02-04T14:01:44.740Z 10 Acked 1009 Flush 0 Done Sent Done false
33512024-02-04T14:01:44.740Z 11 Acked 1010 Write 6 Done Sent Done false
33522024-02-04T14:01:44.740Z 12 Acked 1011 Write 8 Done Sent Done false
33532024-02-04T14:01:44.740Z 13 Acked 1012 Read 1 Done Sent Done false
33542024-02-04T14:01:44.740Z 14 Acked 1013 Read 6 Done Sent Done false
33552024-02-04T14:01:44.740Z 15 Acked 1014 Write 9 Done Sent Done false
33562024-02-04T14:01:44.740Z 16 Acked 1015 Write 1 Done Sent Done false
33572024-02-04T14:01:44.740Z 17 Acked 1016 Read 3 Done Sent Done false
33582024-02-04T14:01:44.740Z 18 Acked 1017 Write 3 Done Sent Done false
33592024-02-04T14:01:44.740Z 19 Acked 1018 Read 5 Done Sent Done false
33602024-02-04T14:01:44.740Z 20 Acked 1019 Flush 0 Done Sent Done false
33612024-02-04T14:01:44.740Z 21 Acked 1020 Write 1 Done Sent Done false
33622024-02-04T14:01:44.740Z 22 Acked 1021 Write 4 Done Sent Done false
33632024-02-04T14:01:44.740Z 23 Acked 1022 Flush 0 Done Sent Done false
33642024-02-04T14:01:44.740Z 24 Acked 1023 Write 8 Done Sent Done false
33652024-02-04T14:01:44.740Z 25 Acked 1024 Read 4 Done Sent Done false
33662024-02-04T14:01:44.740Z 26 Acked 1025 Read 4 Done Sent Done false
33672024-02-04T14:01:44.740Z 27 Acked 1026 Read 8 Done Sent Done false
33682024-02-04T14:01:44.740Z 28 Acked 1027 Write 9 Done Sent Done false
33692024-02-04T14:01:44.740Z 29 Acked 1028 Read 6 Done Sent Done false
33702024-02-04T14:01:44.740Z 30 Acked 1029 Read 8 Done Sent Sent false
33712024-02-04T14:01:44.740Z STATES DS:0 DS:1 DS:2 TOTAL
33722024-02-04T14:01:44.740Z New 0 0 0 0
33732024-02-04T14:01:44.740Z Sent 0 25 1 26
33742024-02-04T14:01:44.740Z Done 28 3 27 58
33752024-02-04T14:01:44.741Z Skipped 0 0 0 0
33762024-02-04T14:01:44.741Z Error 0 0 0 0
33772024-02-04T14:01:44.741ZLast Flush: 1022 1001 1022
33782024-02-04T14:01:44.741ZDownstairs last five completed: 1001 1000
33792024-02-04T14:01:44.741ZUpstairs last five completed: 30 29 28 27 26
33802024-02-04T14:01:44.741Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:43.771570061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1249}
33812024-02-04T14:01:44.741ZWrote out file "/tmp/test_repair_verify.data"
33822024-02-04T14:01:44.745Z
33832024-02-04T14:01:44.752ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33842024-02-04T14:01:44.779ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33852024-02-04T14:01:44.779Z 0 000-019 2 2 2 3 3 3 F F F
33862024-02-04T14:01:44.779Z 1 020-039 24 24 24 40 40 40 F F F
33872024-02-04T14:01:44.779Z 2 040-059 28 28 28 48 48 48 F F F
33882024-02-04T14:01:44.779Z 3 060-079 24 24 24 40 40 40 T F T <---
33892024-02-04T14:01:44.780Z 4 080-099 26 26 26 44 44 44 T F T <---
33902024-02-04T14:01:44.780Z 5 100-119 27 27 27 47 47 47 F F F
33912024-02-04T14:01:44.780Z 6 120-139 25 25 25 43 43 43 F F F
33922024-02-04T14:01:44.780Z 7 140-159 24 24 24 42 42 42 F F F
33932024-02-04T14:01:44.780Z 8 160-179 30 27 30 51 47 51 F F F <---
33942024-02-04T14:01:44.780Z 9 180-199 30 28 30 52 48 52 F F F <---
33952024-02-04T14:01:44.783Z 10 200-219 25 25 25 43 43 43 F F F
33962024-02-04T14:01:44.783Z 11 220-239 30 22 30 52 38 52 F F F <---
33972024-02-04T14:01:44.783Z 12 240-259 30 27 30 51 47 51 F F F <---
33982024-02-04T14:01:44.783Z 13 260-279 28 28 28 48 48 48 F F F
33992024-02-04T14:01:44.783Z 14 280-299 26 26 26 44 44 44 F F F
34002024-02-04T14:01:44.783Z 15 300-319 25 25 25 43 43 43 F F F
34012024-02-04T14:01:44.783Z 16 320-339 27 27 27 47 47 47 F F F
34022024-02-04T14:01:44.783Z 17 340-359 30 30 30 51 49 51 F F F <---
34032024-02-04T14:01:44.783Z 18 360-379 30 30 30 49 49 49 F F F
34042024-02-04T14:01:44.783Z 19 380-399 27 27 27 47 47 47 F F F
34052024-02-04T14:01:44.783Z 20 400-419 30 27 30 50 47 50 F T F <---
34062024-02-04T14:01:44.783Z 21 420-439 28 28 28 48 48 48 F F F
34072024-02-04T14:01:44.783Z 22 440-459 30 15 30 51 23 51 F F F <---
34082024-02-04T14:01:44.783Z 23 460-479 30 28 30 51 48 51 T F T <---
34092024-02-04T14:01:44.784Z 24 480-499 28 28 28 48 48 48 F F F
34102024-02-04T14:01:44.784Z 25 500-519 24 24 24 41 41 41 F F F
34112024-02-04T14:01:44.784Z 26 520-539 30 23 30 51 39 51 F T F <---
34122024-02-04T14:01:44.784Z 27 540-559 30 28 30 50 48 50 F T F <---
34132024-02-04T14:01:44.784Z 28 560-579 30 28 30 50 48 50 F T F <---
34142024-02-04T14:01:44.784Z 29 580-599 30 28 30 50 48 50 F F F <---
34152024-02-04T14:01:44.784ZMax gen: 30, Max flush: 52
34162024-02-04T14:01:44.784ZError: Difference in extent metadata found!
34172024-02-04T14:01:44.787ZA Difference in extent metadata is expected here
34182024-02-04T14:01:44.787ZOn loop 15
34192024-02-04T14:01:44.787Z
34202024-02-04T14:01:44.790ZVerifying data now
34212024-02-04T14:01:51.009ZNo change after verify
34222024-02-04T14:01:51.009ZLoop: 15 Downstairs dump after verify (and repair):
34232024-02-04T14:01:51.032ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34242024-02-04T14:01:51.032Z 0 000-019 2 2 2 3 3 3 F F F
34252024-02-04T14:01:51.032Z 1 020-039 24 24 24 40 40 40 F F F
34262024-02-04T14:01:51.032Z 2 040-059 28 28 28 48 48 48 F F F
34272024-02-04T14:01:51.032Z 3 060-079 31 31 31 53 53 53 F F F
34282024-02-04T14:01:51.032Z 4 080-099 31 31 31 53 53 53 F F F
34292024-02-04T14:01:51.032Z 5 100-119 27 27 27 47 47 47 F F F
34302024-02-04T14:01:51.033Z 6 120-139 25 25 25 43 43 43 F F F
34312024-02-04T14:01:51.033Z 7 140-159 24 24 24 42 42 42 F F F
34322024-02-04T14:01:51.033Z 8 160-179 30 30 30 51 51 51 F F F
34332024-02-04T14:01:51.036Z 9 180-199 30 30 30 52 52 52 F F F
34342024-02-04T14:01:51.036Z 10 200-219 25 25 25 43 43 43 F F F
34352024-02-04T14:01:51.036Z 11 220-239 30 30 30 52 52 52 F F F
34362024-02-04T14:01:51.036Z 12 240-259 30 30 30 51 51 51 F F F
34372024-02-04T14:01:51.036Z 13 260-279 28 28 28 48 48 48 F F F
34382024-02-04T14:01:51.036Z 14 280-299 26 26 26 44 44 44 F F F
34392024-02-04T14:01:51.036Z 15 300-319 25 25 25 43 43 43 F F F
34402024-02-04T14:01:51.036Z 16 320-339 27 27 27 47 47 47 F F F
34412024-02-04T14:01:51.036Z 17 340-359 30 30 30 51 51 51 F F F
34422024-02-04T14:01:51.036Z 18 360-379 30 30 30 49 49 49 F F F
34432024-02-04T14:01:51.037Z 19 380-399 27 27 27 47 47 47 F F F
34442024-02-04T14:01:51.037Z 20 400-419 30 30 30 50 50 50 F F F
34452024-02-04T14:01:51.037Z 21 420-439 28 28 28 48 48 48 F F F
34462024-02-04T14:01:51.037Z 22 440-459 30 30 30 51 51 51 F F F
34472024-02-04T14:01:51.037Z 23 460-479 31 31 31 53 53 53 F F F
34482024-02-04T14:01:51.037Z 24 480-499 28 28 28 48 48 48 F F F
34492024-02-04T14:01:51.037Z 25 500-519 24 24 24 41 41 41 F F F
34502024-02-04T14:01:51.037Z 26 520-539 30 30 30 51 51 51 F F F
34512024-02-04T14:01:51.037Z 27 540-559 30 30 30 50 50 50 F F F
34522024-02-04T14:01:51.037Z 28 560-579 30 30 30 50 50 50 F F F
34532024-02-04T14:01:51.037Z 29 580-599 30 30 30 50 50 50 F F F
34542024-02-04T14:01:51.037ZMax gen: 31, Max flush: 53
34552024-02-04T14:01:51.040Z
34562024-02-04T14:01:51.040ZBegin loop 16
34572024-02-04T14:01:51.040ZDownstairs to restart: 0
34582024-02-04T14:01:51.070Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.103566343Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257}
34592024-02-04T14:01:51.070Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104041044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257}
34602024-02-04T14:01:51.070Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104086008Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257}
34612024-02-04T14:01:51.074Z{"msg":"Crucible c9938b80-7e34-449f-a992-a566d3d9ef60 has session id: 5256b40d-7a61-42ce-b8ca-e95db8eb7afc","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104153866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34622024-02-04T14:01:51.074Z{"msg":"Upstairs opts: Upstairs UUID: c9938b80-7e34-449f-a992-a566d3d9ef60, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104207873Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34632024-02-04T14:01:51.074ZCrucible runtime is spawned
34642024-02-04T14:01:51.074Z{"msg":"Crucible stats registered with UUID: c9938b80-7e34-449f-a992-a566d3d9ef60","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104318646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34652024-02-04T14:01:51.074Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104387201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257}
34662024-02-04T14:01:51.074Z{"msg":"c9938b80-7e34-449f-a992-a566d3d9ef60 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104641301Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34672024-02-04T14:01:51.074Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104702355Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34682024-02-04T14:01:51.074Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.10472762Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34692024-02-04T14:01:51.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.10488632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34702024-02-04T14:01:51.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104929299Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34712024-02-04T14:01:51.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104956715Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34722024-02-04T14:01:51.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.104980467Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34732024-02-04T14:01:51.075Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.105017877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34742024-02-04T14:01:51.075Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.105058906Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34752024-02-04T14:01:51.075Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.105097076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34762024-02-04T14:01:51.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.105315752Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34772024-02-04T14:01:51.075Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.105407304Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34782024-02-04T14:01:51.075Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:50.105531829Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34792024-02-04T14:01:51.075Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:50.105590129Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34802024-02-04T14:01:51.075Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:50.105640492Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34812024-02-04T14:01:51.075Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.105667509Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34822024-02-04T14:01:51.075Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.105694742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34832024-02-04T14:01:51.075Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.105992655Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34842024-02-04T14:01:51.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.106752661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34852024-02-04T14:01:51.075Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.106815462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34862024-02-04T14:01:51.075Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.106841054Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34872024-02-04T14:01:51.075Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.106946972Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34882024-02-04T14:01:51.079Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.107831248Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34892024-02-04T14:01:51.079Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.107918452Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34902024-02-04T14:01:51.079Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.108990031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34912024-02-04T14:01:51.079Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.10905223Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34922024-02-04T14:01:51.079Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.109078564Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34932024-02-04T14:01:51.079Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.10911883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34942024-02-04T14:01:51.079Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.109143117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34952024-02-04T14:01:51.079Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:50.109184207Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34962024-02-04T14:01:52.074Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.10701343Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34972024-02-04T14:01:52.078Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.107244913Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34982024-02-04T14:01:52.078Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.108068961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
34992024-02-04T14:01:52.078Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.108185921Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35002024-02-04T14:01:52.078Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.108893308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35012024-02-04T14:01:52.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.109693756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35022024-02-04T14:01:52.078Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.109776019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35032024-02-04T14:01:52.078Z{"msg":"[0]R flush_numbers[0..12]: [3, 40, 48, 53, 53, 47, 43, 42, 51, 52, 43, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.109849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35042024-02-04T14:01:52.078Z{"msg":"[0]R generation[0..12]: [2, 24, 28, 31, 31, 27, 25, 24, 30, 30, 25, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.109919614Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35052024-02-04T14:01:52.078Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.109992413Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35062024-02-04T14:01:52.078Z{"msg":"Wait for a query_work_queue command to finish before sending IO
35072024-02-04T14:01:52.078Z[1]R flush_numbers[0..12]: [3, 40, 48, 53, 53, 47, 43, 42, 51, 52, 43, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110122331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35082024-02-04T14:01:52.078Z{"msg":"[1]R generation[0..12]: [2, 24, 28, 31, 31, 27, 25, 24, 30, 30, 25, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110188669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35092024-02-04T14:01:52.078Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110260235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35102024-02-04T14:01:52.078Z{"msg":"[2]R flush_numbers[0..12]: [3, 40, 48, 53, 53, 47, 43, 42, 51, 52, 43, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.11030548Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35112024-02-04T14:01:52.078Z{"msg":"[2]R generation[0..12]: [2, 24, 28, 31, 31, 27, 25, 24, 30, 30, 25, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110381996Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35122024-02-04T14:01:52.079Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.11046039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35132024-02-04T14:01:52.079Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110536606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35142024-02-04T14:01:52.079Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110611106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35152024-02-04T14:01:52.079Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35162024-02-04T14:01:52.079Z,"time":"2024-02-04T14:01:51.110686313Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35172024-02-04T14:01:52.079Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110811024Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35182024-02-04T14:01:52.082Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110840722Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35192024-02-04T14:01:52.082Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110881078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35202024-02-04T14:01:52.082Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.11090489Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35212024-02-04T14:01:52.082Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110927834Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35222024-02-04T14:01:52.082Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110968008Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35232024-02-04T14:01:52.082Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.110989866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35242024-02-04T14:01:52.082Z{"msg":"c9938b80-7e34-449f-a992-a566d3d9ef60 is now active with session: 5256b40d-7a61-42ce-b8ca-e95db8eb7afc","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.111028586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35252024-02-04T14:01:52.082Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.11107008Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257,"session_id":"5256b40d-7a61-42ce-b8ca-e95db8eb7afc"}
35262024-02-04T14:01:52.082Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:51.111092775Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257}
35272024-02-04T14:01:52.082ZLoading write count information from file "/tmp/test_repair_verify.data"
35282024-02-04T14:01:52.083ZRun Repair workload
35292024-02-04T14:01:52.083Z01/30 Write block 592 len 4096 data: 2
35302024-02-04T14:01:52.083Z02/30 Read block 205 len 4096
35312024-02-04T14:01:52.083Z03/30 Read block 455 len 4096
35322024-02-04T14:01:52.086Z04/30 Write block 108 len 20480 data: 5 5 5 6 7
35332024-02-04T14:01:52.086Z05/30 Read block 544 len 40960
35342024-02-04T14:01:52.086Z06/30 Flush
35352024-02-04T14:01:52.089Z07/30 Write block 421 len 40960 data: 4 4 4 2 3 3 3 3 3 4
35362024-02-04T14:01:52.092Z08/30 Write block 580 len 20480 data: 5 5 4 5 5
35372024-02-04T14:01:52.092Z09/30 Write block 514 len 36864 data: 4 5 5 3 3 3 2 2 2
35382024-02-04T14:01:52.098Z10/30 Read block 131 len 40960
35392024-02-04T14:01:52.100Z11/30 Write block 96 len 28672 data: 3 3 3 4 3 3 2
35402024-02-04T14:01:52.100Z12/30 Read block 159 len 28672
35412024-02-04T14:01:52.101Z13/30 Read block 465 len 40960
35422024-02-04T14:01:52.101Z14/30 Read block 535 len 12288
35432024-02-04T14:01:52.104Z15/30 Read block 435 len 24576
35442024-02-04T14:01:52.104Z16/30 Read block 595 len 8192
35452024-02-04T14:01:52.106Z17/30 Write block 579 len 40960 data: 5 6 6 5 6 6 5 4 4 3
35462024-02-04T14:01:52.106Z18/30 Write block 413 len 12288 data: 6 6 6
35472024-02-04T14:01:52.106Z19/30 Read block 527 len 8192
35482024-02-04T14:01:52.109Z20/30 Read block 441 len 28672
35492024-02-04T14:01:52.112Z21/30 Write block 282 len 36864 data: 4 4 4 4 4 5 6 6 8
35502024-02-04T14:01:52.112Z22/30 Flush
35512024-02-04T14:01:52.115Z23/30 Write block 294 len 8192 data: 3 3
35522024-02-04T14:01:52.118Z24/30 Write block 516 len 16384 data: 6 4 4 4
35532024-02-04T14:01:52.118Z25/30 Write block 45 len 20480 data: 3 3 2 2 2
35542024-02-04T14:01:52.118Z26/30 Read block 120 len 12288
35552024-02-04T14:01:52.121Z27/30 Write block 325 len 40960 data: 3 3 3 3 3 4 3 3 3 4
35562024-02-04T14:01:52.124Z28/30 Write block 36 len 16384 data: 5 5 3 4
35572024-02-04T14:01:52.124Z29/30 Read block 62 len 28672
35582024-02-04T14:01:52.127Z30/30 Read block 72 len 28672
35592024-02-04T14:01:52.127Z----------------------------------------------------------------
35602024-02-04T14:01:52.127Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35612024-02-04T14:01:52.127ZStates: Active Active Active
35622024-02-04T14:01:52.127ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35632024-02-04T14:01:52.127Z 1 Acked 1000 Write 1 Done Done Done false
35642024-02-04T14:01:52.127Z 2 Acked 1001 Read 1 Sent Done Done false
35652024-02-04T14:01:52.127Z 3 Acked 1002 Read 1 Sent Done Done false
35662024-02-04T14:01:52.127Z 4 Acked 1003 Write 5 Sent Done Done false
35672024-02-04T14:01:52.128Z 5 Acked 1004 Read 10 Sent Done Done false
35682024-02-04T14:01:52.128Z 6 Acked 1005 Flush 0 Sent Done Done false
35692024-02-04T14:01:52.128Z 7 Acked 1006 Write 10 Sent Done Done false
35702024-02-04T14:01:52.128Z 8 Acked 1007 Write 5 Sent Done Done false
35712024-02-04T14:01:52.128Z 9 Acked 1008 Write 9 Sent Done Done false
35722024-02-04T14:01:52.128Z 10 Acked 1009 Read 10 Sent Done Done false
35732024-02-04T14:01:52.128Z 11 Acked 1010 Write 7 Sent Done Done false
35742024-02-04T14:01:52.128Z 12 Acked 1011 Read 7 Sent Done Done false
35752024-02-04T14:01:52.128Z 13 Acked 1012 Read 10 Sent Done Done false
35762024-02-04T14:01:52.128Z 14 Acked 1013 Read 3 Sent Done Done false
35772024-02-04T14:01:52.128Z 15 Acked 1014 Read 6 Sent Done Done false
35782024-02-04T14:01:52.128Z 16 Acked 1015 Read 2 Sent Done Done false
35792024-02-04T14:01:52.128Z 17 Acked 1016 Write 10 Sent Done Done false
35802024-02-04T14:01:52.128Z 18 Acked 1017 Write 3 Sent Done Done false
35812024-02-04T14:01:52.128Z 19 Acked 1018 Read 2 Sent Done Done false
35822024-02-04T14:01:52.128Z 20 Acked 1019 Read 7 Sent Done Done false
35832024-02-04T14:01:52.128Z 21 Acked 1020 Write 9 Sent Done Done false
35842024-02-04T14:01:52.128Z 22 Acked 1021 Flush 0 Sent Done Done false
35852024-02-04T14:01:52.128Z 23 Acked 1022 Write 2 Sent Done Done false
35862024-02-04T14:01:52.128Z 24 Acked 1023 Write 4 Sent Done Done false
35872024-02-04T14:01:52.128Z 25 Acked 1024 Write 5 Sent Done Done false
35882024-02-04T14:01:52.128Z 26 Acked 1025 Read 3 Sent Done Done false
35892024-02-04T14:01:52.128Z 27 Acked 1026 Write 10 Sent Done Done false
35902024-02-04T14:01:52.128Z 28 Acked 1027 Write 4 Sent Done Done false
35912024-02-04T14:01:52.128Z 29 Acked 1028 Read 7 Sent Done Done false
35922024-02-04T14:01:52.129Z 30 Acked 1029 Read 7 Sent Done Done false
35932024-02-04T14:01:52.129Z STATES DS:0 DS:1 DS:2 TOTAL
35942024-02-04T14:01:52.129Z New 0 0 0 0
35952024-02-04T14:01:52.129Z Sent 29 0 0 29
35962024-02-04T14:01:52.129Z Done 1 30 30 61
35972024-02-04T14:01:52.129Z Skipped 0 0 0 0
35982024-02-04T14:01:52.129Z Error 0 0 0 0
35992024-02-04T14:01:52.129ZLast Flush: 0 1021 1021
36002024-02-04T14:01:52.129ZDownstairs last five completed:
36012024-02-04T14:01:52.129ZUpstairs last five completed: 30 29 28 27 26
36022024-02-04T14:01:52.129Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:51.160279211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1257}
36032024-02-04T14:01:52.129ZWrote out file "/tmp/test_repair_verify.data"
36042024-02-04T14:01:52.133Z
36052024-02-04T14:01:52.140ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36062024-02-04T14:01:52.168ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36072024-02-04T14:01:52.168Z 0 000-019 2 2 2 3 3 3 F F F
36082024-02-04T14:01:52.168Z 1 020-039 24 24 24 40 40 40 F T T <---
36092024-02-04T14:01:52.168Z 2 040-059 28 28 28 48 48 48 F T T <---
36102024-02-04T14:01:52.168Z 3 060-079 31 31 31 53 53 53 F F F
36112024-02-04T14:01:52.172Z 4 080-099 31 32 32 53 55 55 F F F <---
36122024-02-04T14:01:52.172Z 5 100-119 27 32 32 47 55 55 F F F <---
36132024-02-04T14:01:52.172Z 6 120-139 25 25 25 43 43 43 F F F
36142024-02-04T14:01:52.172Z 7 140-159 24 24 24 42 42 42 F F F
36152024-02-04T14:01:52.172Z 8 160-179 30 30 30 51 51 51 F F F
36162024-02-04T14:01:52.172Z 9 180-199 30 30 30 52 52 52 F F F
36172024-02-04T14:01:52.172Z 10 200-219 25 25 25 43 43 43 F F F
36182024-02-04T14:01:52.172Z 11 220-239 30 30 30 52 52 52 F F F
36192024-02-04T14:01:52.172Z 12 240-259 30 30 30 51 51 51 F F F
36202024-02-04T14:01:52.172Z 13 260-279 28 28 28 48 48 48 F F F
36212024-02-04T14:01:52.172Z 14 280-299 26 32 32 44 55 55 F T T <---
36222024-02-04T14:01:52.172Z 15 300-319 25 25 25 43 43 43 F F F
36232024-02-04T14:01:52.172Z 16 320-339 27 27 27 47 47 47 F T T <---
36242024-02-04T14:01:52.172Z 17 340-359 30 30 30 51 51 51 F F F
36252024-02-04T14:01:52.172Z 18 360-379 30 30 30 49 49 49 F F F
36262024-02-04T14:01:52.172Z 19 380-399 27 27 27 47 47 47 F F F
36272024-02-04T14:01:52.173Z 20 400-419 30 32 32 50 55 55 F F F <---
36282024-02-04T14:01:52.173Z 21 420-439 28 32 32 48 55 55 F F F <---
36292024-02-04T14:01:52.173Z 22 440-459 30 30 30 51 51 51 F F F
36302024-02-04T14:01:52.173Z 23 460-479 31 31 31 53 53 53 F F F
36312024-02-04T14:01:52.173Z 24 480-499 28 28 28 48 48 48 F F F
36322024-02-04T14:01:52.173Z 25 500-519 24 32 32 41 55 55 F T T <---
36332024-02-04T14:01:52.173Z 26 520-539 30 32 32 51 55 55 F F F <---
36342024-02-04T14:01:52.173Z 27 540-559 30 30 30 50 50 50 F F F
36352024-02-04T14:01:52.173Z 28 560-579 30 32 32 50 55 55 F F F <---
36362024-02-04T14:01:52.173Z 29 580-599 30 32 32 50 55 55 T F F <---
36372024-02-04T14:01:52.173ZMax gen: 32, Max flush: 55
36382024-02-04T14:01:52.173ZError: Difference in extent metadata found!
36392024-02-04T14:01:52.173ZA Difference in extent metadata is expected here
36402024-02-04T14:01:52.173ZOn loop 16
36412024-02-04T14:01:52.173Z
36422024-02-04T14:01:52.176ZVerifying data now
36432024-02-04T14:01:58.260ZNo change after verify
36442024-02-04T14:01:58.260ZLoop: 16 Downstairs dump after verify (and repair):
36452024-02-04T14:01:58.294ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36462024-02-04T14:01:58.297Z 0 000-019 2 2 2 3 3 3 F F F
36472024-02-04T14:01:58.301Z 1 020-039 33 33 33 56 56 56 F F F
36482024-02-04T14:01:58.304Z 2 040-059 33 33 33 56 56 56 F F F
36492024-02-04T14:01:58.306Z 3 060-079 31 31 31 53 53 53 F F F
36502024-02-04T14:01:58.309Z 4 080-099 32 32 32 55 55 55 F F F
36512024-02-04T14:01:58.317Z 5 100-119 32 32 32 55 55 55 F F F
36522024-02-04T14:01:58.317Z 6 120-139 25 25 25 43 43 43 F F F
36532024-02-04T14:01:58.317Z 7 140-159 24 24 24 42 42 42 F F F
36542024-02-04T14:01:58.317Z 8 160-179 30 30 30 51 51 51 F F F
36552024-02-04T14:01:58.317Z 9 180-199 30 30 30 52 52 52 F F F
36562024-02-04T14:01:58.317Z 10 200-219 25 25 25 43 43 43 F F F
36572024-02-04T14:01:58.317Z 11 220-239 30 30 30 52 52 52 F F F
36582024-02-04T14:01:58.317Z 12 240-259 30 30 30 51 51 51 F F F
36592024-02-04T14:01:58.317Z 13 260-279 28 28 28 48 48 48 F F F
36602024-02-04T14:01:58.317Z 14 280-299 33 33 33 56 56 56 F F F
36612024-02-04T14:01:58.318Z 15 300-319 25 25 25 43 43 43 F F F
36622024-02-04T14:01:58.318Z 16 320-339 33 33 33 56 56 56 F F F
36632024-02-04T14:01:58.318Z 17 340-359 30 30 30 51 51 51 F F F
36642024-02-04T14:01:58.318Z 18 360-379 30 30 30 49 49 49 F F F
36652024-02-04T14:01:58.318Z 19 380-399 27 27 27 47 47 47 F F F
36662024-02-04T14:01:58.318Z 20 400-419 32 32 32 55 55 55 F F F
36672024-02-04T14:01:58.318Z 21 420-439 32 32 32 55 55 55 F F F
36682024-02-04T14:01:58.318Z 22 440-459 30 30 30 51 51 51 F F F
36692024-02-04T14:01:58.318Z 23 460-479 31 31 31 53 53 53 F F F
36702024-02-04T14:01:58.318Z 24 480-499 28 28 28 48 48 48 F F F
36712024-02-04T14:01:58.318Z 25 500-519 33 33 33 56 56 56 F F F
36722024-02-04T14:01:58.318Z 26 520-539 32 32 32 55 55 55 F F F
36732024-02-04T14:01:58.319Z 27 540-559 30 30 30 50 50 50 F F F
36742024-02-04T14:01:58.319Z 28 560-579 32 32 32 55 55 55 F F F
36752024-02-04T14:01:58.319Z 29 580-599 32 32 32 55 55 55 F F F
36762024-02-04T14:01:58.319ZMax gen: 33, Max flush: 56
36772024-02-04T14:01:58.319Z
36782024-02-04T14:01:58.319ZBegin loop 17
36792024-02-04T14:01:58.319ZDownstairs to restart: 0
36802024-02-04T14:01:58.327Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.350103469Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265}
36812024-02-04T14:01:58.327Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.350536318Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265}
36822024-02-04T14:01:58.327Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.350564965Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265}
36832024-02-04T14:01:58.328Z{"msg":"Crucible 5d8ab9c8-1c96-4929-aaf9-cb334d8d4e29 has session id: db36f7f2-9419-43a6-8844-21354702d502","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.350620728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36842024-02-04T14:01:58.328ZCrucible runtime is spawned
36852024-02-04T14:01:58.328Z{"msg":"Upstairs opts: Upstairs UUID: 5d8ab9c8-1c96-4929-aaf9-cb334d8d4e29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.350813828Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36862024-02-04T14:01:58.328Z{"msg":"Crucible stats registered with UUID: 5d8ab9c8-1c96-4929-aaf9-cb334d8d4e29","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.350870391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36872024-02-04T14:01:58.328Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.350894119Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265}
36882024-02-04T14:01:58.328Z{"msg":"5d8ab9c8-1c96-4929-aaf9-cb334d8d4e29 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.351011849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36892024-02-04T14:01:58.328Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.351062224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36902024-02-04T14:01:58.328Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.351159356Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36912024-02-04T14:01:58.328Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.351230041Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36922024-02-04T14:01:58.328Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.351343905Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36932024-02-04T14:01:58.328Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.351550014Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36942024-02-04T14:01:58.329Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.351649735Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36952024-02-04T14:01:58.329Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.351764642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36962024-02-04T14:01:58.329Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.352046068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36972024-02-04T14:01:58.329Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.352212482Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36982024-02-04T14:01:58.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.352287996Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
36992024-02-04T14:01:58.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.352333596Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37002024-02-04T14:01:58.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.35240145Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37012024-02-04T14:01:58.329Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.352779874Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37022024-02-04T14:01:58.329Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.352948574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37032024-02-04T14:01:58.329Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.353076234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37042024-02-04T14:01:58.329Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.353155556Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37052024-02-04T14:01:58.329Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.353308549Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37062024-02-04T14:01:58.329Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.353450006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37072024-02-04T14:01:58.330Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.353856939Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37082024-02-04T14:01:58.330Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.353947063Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37092024-02-04T14:01:58.330Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.354404705Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37102024-02-04T14:01:58.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.354709218Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37112024-02-04T14:01:58.330Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.354766515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37122024-02-04T14:01:58.330Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.354790279Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37132024-02-04T14:01:58.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.354811434Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37142024-02-04T14:01:58.330Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.354835204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37152024-02-04T14:01:58.330Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.354860675Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37162024-02-04T14:01:58.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355117729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37172024-02-04T14:01:58.330Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355171759Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37182024-02-04T14:01:58.330Z{"msg":"[0]R flush_numbers[0..12]: [3, 56, 56, 53, 55, 55, 43, 42, 51, 52, 43, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355197673Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37192024-02-04T14:01:58.331Z{"msg":"[0]R generation[0..12]: [2, 33, 33, 31, 32, 32, 25, 24, 30, 30, 25, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355239955Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37202024-02-04T14:01:58.331Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355265456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37212024-02-04T14:01:58.331Z{"msg":"[1]R flush_numbers[0..12]: [3, 56, 56, 53, 55, 55, 43, 42, 51, 52, 43, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355307742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37222024-02-04T14:01:58.331Z{"msg":"[1]R generation[0..12]: [2, 33, 33, 31, 32, 32, 25, 24, 30, 30, 25, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355331874Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37232024-02-04T14:01:58.331Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355357259Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37242024-02-04T14:01:58.331ZWait for a query_work_queue command to finish before sending IO
37252024-02-04T14:01:58.331Z{"msg":"[2]R flush_numbers[0..12]: [3, 56, 56, 53, 55, 55, 43, 42, 51, 52, 43, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355400531Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37262024-02-04T14:01:58.331Z{"msg":"[2]R generation[0..12]: [2, 33, 33, 31, 32, 32, 25, 24, 30, 30, 25, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355424679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37272024-02-04T14:01:58.331Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355477515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37282024-02-04T14:01:58.331Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355500486Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37292024-02-04T14:01:58.331Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.35554068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37302024-02-04T14:01:58.331Z{"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355563558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37312024-02-04T14:01:58.332Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355604323Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37322024-02-04T14:01:58.332Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355627843Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37332024-02-04T14:01:58.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355666501Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37342024-02-04T14:01:58.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355689413Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37352024-02-04T14:01:58.332Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355711033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37362024-02-04T14:01:58.332Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355751493Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37372024-02-04T14:01:58.332Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355773301Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37382024-02-04T14:01:58.332Z{"msg":"5d8ab9c8-1c96-4929-aaf9-cb334d8d4e29 is now active with session: db36f7f2-9419-43a6-8844-21354702d502","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.35581326Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37392024-02-04T14:01:58.332Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355834456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265,"session_id":"db36f7f2-9419-43a6-8844-21354702d502"}
37402024-02-04T14:01:58.332Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:01:57.355877337Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265}
37412024-02-04T14:01:58.332ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37422024-02-04T14:01:58.332ZLoading write count information from file "/tmp/test_repair_verify.data"
37432024-02-04T14:01:58.332ZRun Repair workload
37442024-02-04T14:01:58.333Z01/30 Write block 420 len 28672 data: 5 5 5 5 3 4 4
37452024-02-04T14:01:58.337Z02/30 Read block 141 len 16384
37462024-02-04T14:01:58.337Z03/30 Read block 407 len 4096
37472024-02-04T14:01:58.337Z04/30 Read block 588 len 4096
37482024-02-04T14:01:58.337Z05/30 Write block 478 len 12288 data: 3 3 3
37492024-02-04T14:01:58.337Z06/30 Write block 291 len 8192 data: 6 6
37502024-02-04T14:01:58.338Z07/30 Read block 59 len 20480
37512024-02-04T14:01:58.338Z08/30 Flush
37522024-02-04T14:01:58.341Z09/30 Write block 4 len 16384 data: 2 2 2 2
37532024-02-04T14:01:58.341Z10/30 Read block 389 len 24576
37542024-02-04T14:01:58.342Z11/30 Write block 377 len 8192 data: 3 4
37552024-02-04T14:01:58.342Z12/30 Read block 257 len 36864
37562024-02-04T14:01:58.347Z13/30 Write block 392 len 36864 data: 6 5 4 2 2 2 3 3 4
37572024-02-04T14:01:58.347Z14/30 Read block 459 len 20480
37582024-02-04T14:01:58.347Z15/30 Read block 389 len 12288
37592024-02-04T14:01:58.351Z16/30 Read block 575 len 32768
37602024-02-04T14:01:58.351Z17/30 Write block 95 len 4096 data: 3
37612024-02-04T14:01:58.351Z18/30 Read block 248 len 4096
37622024-02-04T14:01:58.351Z19/30 Read block 496 len 4096
37632024-02-04T14:01:58.356Z20/30 Write block 397 len 4096 data: 3
37642024-02-04T14:01:58.359Z21/30 Read block 134 len 12288
37652024-02-04T14:01:58.359Z22/30 Read block 424 len 4096
37662024-02-04T14:01:58.359Z23/30 Read block 517 len 4096
37672024-02-04T14:01:58.366Z24/30 Write block 79 len 36864 data: 4 4 5 5 4 5 5 4 4
37682024-02-04T14:01:58.366Z25/30 Write block 450 len 12288 data: 3 3 4
37692024-02-04T14:01:58.366Z26/30 Write block 492 len 36864 data: 6 6 5 5 6 5 6 6 6
37702024-02-04T14:01:58.366Z27/30 Write block 225 len 24576 data: 2 2 3 3 2 2
37712024-02-04T14:01:58.366Z28/30 Read block 239 len 40960
37722024-02-04T14:01:58.367Z29/30 Write block 422 len 8192 data: 6 6
37732024-02-04T14:01:58.367Z30/30 Write block 221 len 40960 data: 3 3 4 3 3 3 4 4 3 3
37742024-02-04T14:01:58.407Z----------------------------------------------------------------
37752024-02-04T14:01:58.407Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37762024-02-04T14:01:58.407ZStates: Active Active Active
37772024-02-04T14:01:58.407ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37782024-02-04T14:01:58.407Z 1 Acked 1000 Write 7 Done Done Done false
37792024-02-04T14:01:58.407Z 2 Acked 1001 Read 4 Done Done Done false
37802024-02-04T14:01:58.408Z 3 Acked 1002 Read 1 Sent Done Done false
37812024-02-04T14:01:58.408Z 4 Acked 1003 Read 1 Sent Done Done false
37822024-02-04T14:01:58.408Z 5 Acked 1004 Write 3 Sent Done Done false
37832024-02-04T14:01:58.408Z 6 Acked 1005 Write 2 Sent Done Done false
37842024-02-04T14:01:58.408Z 7 Acked 1006 Read 5 Sent Done Done false
37852024-02-04T14:01:58.408Z 8 Acked 1007 Flush 0 Sent Done Done false
37862024-02-04T14:01:58.408Z 9 Acked 1008 Write 4 Sent Done Done false
37872024-02-04T14:01:58.408Z 10 Acked 1009 Read 6 Sent Done Done false
37882024-02-04T14:01:58.408Z 11 Acked 1010 Write 2 Sent Done Done false
37892024-02-04T14:01:58.408Z 12 Acked 1011 Read 9 Sent Done Done false
37902024-02-04T14:01:58.408Z 13 Acked 1012 Write 9 Sent Done Done false
37912024-02-04T14:01:58.408Z 14 Acked 1013 Read 5 Sent Done Done false
37922024-02-04T14:01:58.408Z 15 Acked 1014 Read 3 Sent Done Done false
37932024-02-04T14:01:58.408Z 16 Acked 1015 Read 8 Sent Done Done false
37942024-02-04T14:01:58.409Z 17 Acked 1016 Write 1 Sent Done Done false
37952024-02-04T14:01:58.409Z 18 Acked 1017 Read 1 Sent Done Done false
37962024-02-04T14:01:58.409Z 19 Acked 1018 Read 1 Sent Done Done false
37972024-02-04T14:01:58.409Z 20 Acked 1019 Write 1 Sent Done Done false
37982024-02-04T14:01:58.409Z 21 Acked 1020 Read 3 Sent Done Done false
37992024-02-04T14:01:58.409Z 22 Acked 1021 Read 1 Sent Done Done false
38002024-02-04T14:01:58.409Z 23 Acked 1022 Read 1 Sent Done Done false
38012024-02-04T14:01:58.409Z 24 Acked 1023 Write 9 Sent Done Done false
38022024-02-04T14:01:58.409Z 25 Acked 1024 Write 3 Sent Done Done false
38032024-02-04T14:01:58.409Z 26 Acked 1025 Write 9 Sent Done Done false
38042024-02-04T14:01:58.409Z 27 Acked 1026 Write 6 Sent Done Done false
38052024-02-04T14:01:58.409Z 28 Acked 1027 Read 10 Sent Done Done false
38062024-02-04T14:01:58.409Z 29 Acked 1028 Write 2 Sent Done Done false
38072024-02-04T14:01:58.409Z 30 Acked 1029 Write 10 Sent Sent Sent false
38082024-02-04T14:01:58.409Z STATES DS:0 DS:1 DS:2 TOTAL
38092024-02-04T14:01:58.410Z New 0 0 0 0
38102024-02-04T14:01:58.410Z Sent 28 1 1 30
38112024-02-04T14:01:58.410Z Done 2 29 29 60
38122024-02-04T14:01:58.410Z Skipped 0 0 0 0
38132024-02-04T14:01:58.410Z Error 0 0 0 0
38142024-02-04T14:01:58.410ZLast Flush: 0 1007 1007
38152024-02-04T14:01:58.410ZDownstairs last five completed:
38162024-02-04T14:01:58.410ZUpstairs last five completed: 30 29 28 27 26
38172024-02-04T14:01:58.410Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:01:57.398419017Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1265}
38182024-02-04T14:01:58.410ZWrote out file "/tmp/test_repair_verify.data"
38192024-02-04T14:01:58.410Z
38202024-02-04T14:01:58.410ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38212024-02-04T14:01:58.410ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38222024-02-04T14:01:58.410Z 0 000-019 2 2 2 3 3 3 F T T <---
38232024-02-04T14:01:58.410Z 1 020-039 33 33 33 56 56 56 F F F
38242024-02-04T14:01:58.411Z 2 040-059 33 33 33 56 56 56 F F F
38252024-02-04T14:01:58.411Z 3 060-079 31 31 31 53 53 53 F T T <---
38262024-02-04T14:01:58.411Z 4 080-099 32 32 32 55 55 55 F T T <---
38272024-02-04T14:01:58.411Z 5 100-119 32 32 32 55 55 55 F F F
38282024-02-04T14:01:58.411Z 6 120-139 25 25 25 43 43 43 F F F
38292024-02-04T14:01:58.411Z 7 140-159 24 24 24 42 42 42 F F F
38302024-02-04T14:01:58.411Z 8 160-179 30 30 30 51 51 51 F F F
38312024-02-04T14:01:58.412Z 9 180-199 30 30 30 52 52 52 F F F
38322024-02-04T14:01:58.412Z 10 200-219 25 25 25 43 43 43 F F F
38332024-02-04T14:01:58.412Z 11 220-239 30 30 30 52 52 52 F T T <---
38342024-02-04T14:01:58.412Z 12 240-259 30 30 30 51 51 51 F F F
38352024-02-04T14:01:58.412Z 13 260-279 28 28 28 48 48 48 F F F
38362024-02-04T14:01:58.412Z 14 280-299 33 34 34 56 57 57 F F F <---
38372024-02-04T14:01:58.412Z 15 300-319 25 25 25 43 43 43 F F F
38382024-02-04T14:01:58.412Z 16 320-339 33 33 33 56 56 56 F F F
38392024-02-04T14:01:58.412Z 17 340-359 30 30 30 51 51 51 F F F
38402024-02-04T14:01:58.413Z 18 360-379 30 30 30 49 49 49 F T T <---
38412024-02-04T14:01:58.413Z 19 380-399 27 27 27 47 47 47 F T T <---
38422024-02-04T14:01:58.413Z 20 400-419 32 32 32 55 55 55 F T T <---
38432024-02-04T14:01:58.413Z 21 420-439 32 34 34 55 57 57 T T T <---
38442024-02-04T14:01:58.413Z 22 440-459 30 30 30 51 51 51 F T T <---
38452024-02-04T14:01:58.413Z 23 460-479 31 34 34 53 57 57 F F F <---
38462024-02-04T14:01:58.413Z 24 480-499 28 34 34 48 57 57 F T T <---
38472024-02-04T14:01:58.413Z 25 500-519 33 33 33 56 56 56 F T T <---
38482024-02-04T14:01:58.413Z 26 520-539 32 32 32 55 55 55 F F F
38492024-02-04T14:01:58.413Z 27 540-559 30 30 30 50 50 50 F F F
38502024-02-04T14:01:58.413Z 28 560-579 32 32 32 55 55 55 F F F
38512024-02-04T14:01:58.413Z 29 580-599 32 32 32 55 55 55 F F F
38522024-02-04T14:01:58.413ZMax gen: 34, Max flush: 57
38532024-02-04T14:01:58.413ZError: Difference in extent metadata found!
38542024-02-04T14:01:58.419ZA Difference in extent metadata is expected here
38552024-02-04T14:01:58.419ZOn loop 17
38562024-02-04T14:01:58.419Z
38572024-02-04T14:01:58.419ZVerifying data now
38582024-02-04T14:02:03.565ZNo change after verify
38592024-02-04T14:02:03.565ZLoop: 17 Downstairs dump after verify (and repair):
38602024-02-04T14:02:03.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38612024-02-04T14:02:03.588Z 0 000-019 35 35 35 58 58 58 F F F
38622024-02-04T14:02:03.588Z 1 020-039 33 33 33 56 56 56 F F F
38632024-02-04T14:02:03.588Z 2 040-059 33 33 33 56 56 56 F F F
38642024-02-04T14:02:03.588Z 3 060-079 35 35 35 58 58 58 F F F
38652024-02-04T14:02:03.588Z 4 080-099 35 35 35 58 58 58 F F F
38662024-02-04T14:02:03.588Z 5 100-119 32 32 32 55 55 55 F F F
38672024-02-04T14:02:03.588Z 6 120-139 25 25 25 43 43 43 F F F
38682024-02-04T14:02:03.588Z 7 140-159 24 24 24 42 42 42 F F F
38692024-02-04T14:02:03.588Z 8 160-179 30 30 30 51 51 51 F F F
38702024-02-04T14:02:03.588Z 9 180-199 30 30 30 52 52 52 F F F
38712024-02-04T14:02:03.588Z 10 200-219 25 25 25 43 43 43 F F F
38722024-02-04T14:02:03.588Z 11 220-239 35 35 35 58 58 58 F F F
38732024-02-04T14:02:03.588Z 12 240-259 30 30 30 51 51 51 F F F
38742024-02-04T14:02:03.588Z 13 260-279 28 28 28 48 48 48 F F F
38752024-02-04T14:02:03.597Z 14 280-299 34 34 34 57 57 57 F F F
38762024-02-04T14:02:03.597Z 15 300-319 25 25 25 43 43 43 F F F
38772024-02-04T14:02:03.597Z 16 320-339 33 33 33 56 56 56 F F F
38782024-02-04T14:02:03.597Z 17 340-359 30 30 30 51 51 51 F F F
38792024-02-04T14:02:03.597Z 18 360-379 35 35 35 58 58 58 F F F
38802024-02-04T14:02:03.597Z 19 380-399 35 35 35 58 58 58 F F F
38812024-02-04T14:02:03.597Z 20 400-419 35 35 35 58 58 58 F F F
38822024-02-04T14:02:03.597Z 21 420-439 35 35 35 58 58 58 F F F
38832024-02-04T14:02:03.597Z 22 440-459 35 35 35 58 58 58 F F F
38842024-02-04T14:02:03.597Z 23 460-479 34 34 34 57 57 57 F F F
38852024-02-04T14:02:03.597Z 24 480-499 35 35 35 58 58 58 F F F
38862024-02-04T14:02:03.597Z 25 500-519 35 35 35 58 58 58 F F F
38872024-02-04T14:02:03.598Z 26 520-539 32 32 32 55 55 55 F F F
38882024-02-04T14:02:03.598Z 27 540-559 30 30 30 50 50 50 F F F
38892024-02-04T14:02:03.598Z 28 560-579 32 32 32 55 55 55 F F F
38902024-02-04T14:02:03.598Z 29 580-599 32 32 32 55 55 55 F F F
38912024-02-04T14:02:03.598ZMax gen: 35, Max flush: 58
38922024-02-04T14:02:03.598Z
38932024-02-04T14:02:03.598ZBegin loop 18
38942024-02-04T14:02:03.598ZDownstairs to restart: 1
38952024-02-04T14:02:03.622Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.654909549Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273}
38962024-02-04T14:02:03.622Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.655441029Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273}
38972024-02-04T14:02:03.623Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.655475348Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273}
38982024-02-04T14:02:03.629Z{"msg":"Crucible e059925d-2304-4163-a2e1-b76df127d84e has session id: 8a9efc1b-580f-4f4f-94e9-415a9c6b5435","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.655543286Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
38992024-02-04T14:02:03.629Z{"msg":"Upstairs opts: Upstairs UUID: e059925d-2304-4163-a2e1-b76df127d84e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.655661861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39002024-02-04T14:02:03.629Z{"msg":"Crucible stats registered with UUID: e059925d-2304-4163-a2e1-b76df127d84e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.655728139Z","hostname"Crucible runtime is spawned
39012024-02-04T14:02:03.629Z:"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39022024-02-04T14:02:03.629Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.655880551Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273}
39032024-02-04T14:02:03.629Z{"msg":"e059925d-2304-4163-a2e1-b76df127d84e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656071586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39042024-02-04T14:02:03.629Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656138519Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39052024-02-04T14:02:03.629Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656166149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39062024-02-04T14:02:03.629Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656232499Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39072024-02-04T14:02:03.629Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656260637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39082024-02-04T14:02:03.629Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656321528Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39092024-02-04T14:02:03.629Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.65635983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39102024-02-04T14:02:03.630Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656439367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"io task","client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39112024-02-04T14:02:03.630Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656648405Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"io task","client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39122024-02-04T14:02:03.630Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656741318Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"io task","client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39132024-02-04T14:02:03.630Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656785097Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"io task","client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39142024-02-04T14:02:03.630Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656824738Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"io task","client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39152024-02-04T14:02:03.630Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.656915973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"io task","client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39162024-02-04T14:02:03.630Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.658002455Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39172024-02-04T14:02:03.630Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.658120222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39182024-02-04T14:02:03.630Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.658177464Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39192024-02-04T14:02:03.630Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.658225606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39202024-02-04T14:02:03.630Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.658413979Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39212024-02-04T14:02:03.630Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.658499973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39222024-02-04T14:02:03.630Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.659139003Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39232024-02-04T14:02:03.630Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.659252053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39242024-02-04T14:02:03.630Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.659361709Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39252024-02-04T14:02:03.630Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.66007352Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39262024-02-04T14:02:03.630Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660158445Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39272024-02-04T14:02:03.630Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660235413Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39282024-02-04T14:02:03.630Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.66027746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39292024-02-04T14:02:03.630Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.66033973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39302024-02-04T14:02:03.630Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660372458Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39312024-02-04T14:02:03.630Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660426373Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39322024-02-04T14:02:03.631Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660457882Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39332024-02-04T14:02:03.631Z{"msg":"[0]R flush_numbers[0..12]: [58, 56, 56, 58, 58, 55, 43, 42, 51, 52, 43, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660516996Z","hostname":"Wait for a query_work_queue command to finish before sending IO
39342024-02-04T14:02:03.631Zip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39352024-02-04T14:02:03.631Z{"msg":"[0]R generation[0..12]: [35, 33, 33, 35, 35, 32, 25, 24, 30, 30, 25, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660551257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39362024-02-04T14:02:03.631Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660595372Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39372024-02-04T14:02:03.631Z{"msg":"[1]R flush_numbers[0..12]: [58, 56, 56, 58, 58, 55, 43, 42, 51, 52, 43, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660635758Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39382024-02-04T14:02:03.631Z{"msg":"[1]R generation[0..12]: [35, 33, 33, 35, 35, 32, 25, 24, 30, 30, 25, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660704165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39392024-02-04T14:02:03.631Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660752059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39402024-02-04T14:02:03.631Z{"msg":"[2]R flush_numbers[0..12]: [58, 56, 56, 58, 58, 55, 43, 42, 51, 52, 43, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.66078242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39412024-02-04T14:02:03.631Z{"msg":"[2]R generation[0..12]: [35, 33, 33, 35, 35, 32, 25, 24, 30, 30, 25, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660807738Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39422024-02-04T14:02:03.631Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660831793Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39432024-02-04T14:02:03.631Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660853806Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39442024-02-04T14:02:03.641Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660876511Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39452024-02-04T14:02:03.641Z{"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660898231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39462024-02-04T14:02:03.641Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660920292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39472024-02-04T14:02:03.641Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660952142Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39482024-02-04T14:02:03.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.660974353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39492024-02-04T14:02:03.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.661012087Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39502024-02-04T14:02:03.641Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.661041472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39512024-02-04T14:02:03.641Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.661077897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39522024-02-04T14:02:03.641Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.661134898Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39532024-02-04T14:02:03.641Z{"msg":"e059925d-2304-4163-a2e1-b76df127d84e is now active with session: 8a9efc1b-580f-4f4f-94e9-415a9c6b5435","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.661161213Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39542024-02-04T14:02:03.641Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.661186836Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273,"session_id":"8a9efc1b-580f-4f4f-94e9-415a9c6b5435"}
39552024-02-04T14:02:03.641Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:02.661251036Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273}
39562024-02-04T14:02:03.641ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39572024-02-04T14:02:03.641ZLoading write count information from file "/tmp/test_repair_verify.data"
39582024-02-04T14:02:03.641ZRun Repair workload
39592024-02-04T14:02:03.641Z01/30 Write block 374 len 8192 data: 3 3
39602024-02-04T14:02:03.641Z02/30 Read block 379 len 28672
39612024-02-04T14:02:03.641Z03/30 Write block 249 len 4096 data: 5
39622024-02-04T14:02:03.641Z04/30 Read block 84 len 36864
39632024-02-04T14:02:03.641Z05/30 Read block 278 len 28672
39642024-02-04T14:02:03.641Z06/30 Write block 489 len 20480 data: 6 6 6 7 7
39652024-02-04T14:02:03.645Z07/30 Read block 536 len 20480
39662024-02-04T14:02:03.645Z08/30 Flush
39672024-02-04T14:02:03.645Z09/30 Write block 461 len 40960 data: 5 4 4 4 4 5 4 3 4 5
39682024-02-04T14:02:03.645Z10/30 Write block 292 len 8192 data: 7 3
39692024-02-04T14:02:03.645Z11/30 Write block 104 len 16384 data: 3 3 3 3
39702024-02-04T14:02:03.649Z12/30 Read block 6 len 4096
39712024-02-04T14:02:03.649Z13/30 Write block 170 len 32768 data: 6 6 7 7 8 7 6 6
39722024-02-04T14:02:03.649Z14/30 Read block 513 len 20480
39732024-02-04T14:02:03.654Z15/30 Write block 77 len 12288 data: 4 4 5
39742024-02-04T14:02:03.654Z16/30 Write block 51 len 12288 data: 2 2 2
39752024-02-04T14:02:03.654Z17/30 Read block 225 len 24576
39762024-02-04T14:02:03.654Z18/30 Flush
39772024-02-04T14:02:03.659Z19/30 Write block 355 len 16384 data: 4 4 3 3
39782024-02-04T14:02:03.659Z20/30 Read block 321 len 24576
39792024-02-04T14:02:03.666Z21/30 Write block 278 len 32768 data: 5 5 4 4 5 5 5 5
39802024-02-04T14:02:03.666Z22/30 Write block 491 len 24576 data: 7 8 8 6 6 7
39812024-02-04T14:02:03.666Z23/30 Write block 405 len 28672 data: 3 3 3 3 4 4 6
39822024-02-04T14:02:03.666Z24/30 Read block 474 len 4096
39832024-02-04T14:02:03.666Z25/30 Read block 381 len 12288
39842024-02-04T14:02:03.670Z26/30 Write block 214 len 8192 data: 4 5
39852024-02-04T14:02:03.670Z27/30 Read block 536 len 16384
39862024-02-04T14:02:03.670Z28/30 Read block 371 len 16384
39872024-02-04T14:02:03.675Z29/30 Write block 172 len 8192 data: 8 8
39882024-02-04T14:02:03.675Z30/30 Read block 22 len 20480
39892024-02-04T14:02:03.675Z----------------------------------------------------------------
39902024-02-04T14:02:03.675Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
39912024-02-04T14:02:03.675ZStates: Active Active Active
39922024-02-04T14:02:03.675ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39932024-02-04T14:02:03.675Z 1 Acked 1000 Write 2 Done Done Done false
39942024-02-04T14:02:03.676Z 2 Acked 1001 Read 7 Done Done Done false
39952024-02-04T14:02:03.676Z 3 Acked 1002 Write 1 Done Done Done false
39962024-02-04T14:02:03.676Z 4 Acked 1003 Read 9 Done Done Done false
39972024-02-04T14:02:03.676Z 5 Acked 1004 Read 7 Done Sent Done false
39982024-02-04T14:02:03.676Z 6 Acked 1005 Write 5 Done Sent Done false
39992024-02-04T14:02:03.676Z 7 Acked 1006 Read 5 Done Sent Done false
40002024-02-04T14:02:03.676Z 8 Acked 1007 Flush 0 Done Sent Done false
40012024-02-04T14:02:03.676Z 9 Acked 1008 Write 10 Done Sent Done false
40022024-02-04T14:02:03.676Z 10 Acked 1009 Write 2 Done Sent Done false
40032024-02-04T14:02:03.676Z 11 Acked 1010 Write 4 Done Sent Done false
40042024-02-04T14:02:03.676Z 12 Acked 1011 Read 1 Done Sent Done false
40052024-02-04T14:02:03.676Z 13 Acked 1012 Write 8 Done Sent Done false
40062024-02-04T14:02:03.676Z 14 Acked 1013 Read 5 Done Sent Done false
40072024-02-04T14:02:03.676Z 15 Acked 1014 Write 3 Done Sent Done false
40082024-02-04T14:02:03.676Z 16 Acked 1015 Write 3 Done Sent Done false
40092024-02-04T14:02:03.676Z 17 Acked 1016 Read 6 Done Sent Done false
40102024-02-04T14:02:03.676Z 18 Acked 1017 Flush 0 Done Sent Done false
40112024-02-04T14:02:03.676Z 19 Acked 1018 Write 4 Done Sent Done false
40122024-02-04T14:02:03.676Z 20 Acked 1019 Read 6 Done Sent Done false
40132024-02-04T14:02:03.676Z 21 Acked 1020 Write 8 Done Sent Done false
40142024-02-04T14:02:03.677Z 22 Acked 1021 Write 6 Done Sent Done false
40152024-02-04T14:02:03.677Z 23 Acked 1022 Write 7 Done Sent Done false
40162024-02-04T14:02:03.677Z 24 Acked 1023 Read 1 Done Sent Done false
40172024-02-04T14:02:03.677Z 25 Acked 1024 Read 3 Done Sent Done false
40182024-02-04T14:02:03.677Z 26 Acked 1025 Write 2 Done Sent Done false
40192024-02-04T14:02:03.677Z 27 Acked 1026 Read 4 Done Sent Done false
40202024-02-04T14:02:03.677Z 28 Acked 1027 Read 4 Done Sent Done false
40212024-02-04T14:02:03.677Z 29 Acked 1028 Write 2 Done Sent Done false
40222024-02-04T14:02:03.677Z 30 Acked 1029 Read 5 Done Sent Done false
40232024-02-04T14:02:03.677Z STATES DS:0 DS:1 DS:2 TOTAL
40242024-02-04T14:02:03.677Z New 0 0 0 0
40252024-02-04T14:02:03.677Z Sent 0 26 0 26
40262024-02-04T14:02:03.677Z Done 30 4 30 64
40272024-02-04T14:02:03.677Z Skipped 0 0 0 0
40282024-02-04T14:02:03.677Z Error 0 0 0 0
40292024-02-04T14:02:03.677ZLast Flush: 1017 0 1017
40302024-02-04T14:02:03.677ZDownstairs last five completed:
40312024-02-04T14:02:03.677ZUpstairs last five completed: 30 29 28 27 26
40322024-02-04T14:02:03.677Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:02.706441567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1273}
40332024-02-04T14:02:03.677ZWrote out file "/tmp/test_repair_verify.data"
40342024-02-04T14:02:03.685Z
40352024-02-04T14:02:03.691ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40362024-02-04T14:02:03.714ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40372024-02-04T14:02:03.714Z 0 000-019 35 35 35 58 58 58 F F F
40382024-02-04T14:02:03.714Z 1 020-039 33 33 33 56 56 56 F F F
40392024-02-04T14:02:03.714Z 2 040-059 36 33 36 60 56 60 F F F <---
40402024-02-04T14:02:03.714Z 3 060-079 36 35 36 60 58 60 F F F <---
40412024-02-04T14:02:03.714Z 4 080-099 35 35 35 58 58 58 F F F
40422024-02-04T14:02:03.714Z 5 100-119 36 32 36 60 55 60 F F F <---
40432024-02-04T14:02:03.718Z 6 120-139 25 25 25 43 43 43 F F F
40442024-02-04T14:02:03.718Z 7 140-159 24 24 24 42 42 42 F F F
40452024-02-04T14:02:03.718Z 8 160-179 36 30 36 60 51 60 T F T <---
40462024-02-04T14:02:03.718Z 9 180-199 30 30 30 52 52 52 F F F
40472024-02-04T14:02:03.718Z 10 200-219 25 25 25 43 43 43 T F T <---
40482024-02-04T14:02:03.718Z 11 220-239 35 35 35 58 58 58 F F F
40492024-02-04T14:02:03.718Z 12 240-259 36 30 36 59 51 59 F T F <---
40502024-02-04T14:02:03.718Z 13 260-279 28 28 28 48 48 48 T F T <---
40512024-02-04T14:02:03.718Z 14 280-299 36 34 36 60 57 60 T F T <---
40522024-02-04T14:02:03.718Z 15 300-319 25 25 25 43 43 43 F F F
40532024-02-04T14:02:03.718Z 16 320-339 33 33 33 56 56 56 F F F
40542024-02-04T14:02:03.718Z 17 340-359 30 30 30 51 51 51 T F T <---
40552024-02-04T14:02:03.718Z 18 360-379 36 35 36 59 58 59 F T F <---
40562024-02-04T14:02:03.718Z 19 380-399 35 35 35 58 58 58 F F F
40572024-02-04T14:02:03.718Z 20 400-419 35 35 35 58 58 58 T F T <---
40582024-02-04T14:02:03.718Z 21 420-439 35 35 35 58 58 58 F F F
40592024-02-04T14:02:03.718Z 22 440-459 35 35 35 58 58 58 F F F
40602024-02-04T14:02:03.718Z 23 460-479 36 34 36 60 57 60 F F F <---
40612024-02-04T14:02:03.718Z 24 480-499 36 35 36 59 58 59 T F T <---
40622024-02-04T14:02:03.718Z 25 500-519 35 35 35 58 58 58 F F F
40632024-02-04T14:02:03.719Z 26 520-539 32 32 32 55 55 55 F F F
40642024-02-04T14:02:03.719Z 27 540-559 30 30 30 50 50 50 F F F
40652024-02-04T14:02:03.719Z 28 560-579 32 32 32 55 55 55 F F F
40662024-02-04T14:02:03.719Z 29 580-599 32 32 32 55 55 55 F F F
40672024-02-04T14:02:03.719ZMax gen: 36, Max flush: 60
40682024-02-04T14:02:03.719ZError: Difference in extent metadata found!
40692024-02-04T14:02:03.719ZA Difference in extent metadata is expected here
40702024-02-04T14:02:03.719ZOn loop 18
40712024-02-04T14:02:03.719Z
40722024-02-04T14:02:03.722ZVerifying data now
40732024-02-04T14:02:09.866ZNo change after verify
40742024-02-04T14:02:09.867ZLoop: 18 Downstairs dump after verify (and repair):
40752024-02-04T14:02:09.889ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40762024-02-04T14:02:09.889Z 0 000-019 35 35 35 58 58 58 F F F
40772024-02-04T14:02:09.889Z 1 020-039 33 33 33 56 56 56 F F F
40782024-02-04T14:02:09.889Z 2 040-059 36 36 36 60 60 60 F F F
40792024-02-04T14:02:09.890Z 3 060-079 36 36 36 60 60 60 F F F
40802024-02-04T14:02:09.890Z 4 080-099 35 35 35 58 58 58 F F F
40812024-02-04T14:02:09.890Z 5 100-119 36 36 36 60 60 60 F F F
40822024-02-04T14:02:09.890Z 6 120-139 25 25 25 43 43 43 F F F
40832024-02-04T14:02:09.890Z 7 140-159 24 24 24 42 42 42 F F F
40842024-02-04T14:02:09.890Z 8 160-179 37 37 37 61 61 61 F F F
40852024-02-04T14:02:09.890Z 9 180-199 30 30 30 52 52 52 F F F
40862024-02-04T14:02:09.890Z 10 200-219 37 37 37 61 61 61 F F F
40872024-02-04T14:02:09.890Z 11 220-239 35 35 35 58 58 58 F F F
40882024-02-04T14:02:09.893Z 12 240-259 36 36 36 59 59 59 F F F
40892024-02-04T14:02:09.893Z 13 260-279 37 37 37 61 61 61 F F F
40902024-02-04T14:02:09.893Z 14 280-299 37 37 37 61 61 61 F F F
40912024-02-04T14:02:09.893Z 15 300-319 25 25 25 43 43 43 F F F
40922024-02-04T14:02:09.893Z 16 320-339 33 33 33 56 56 56 F F F
40932024-02-04T14:02:09.893Z 17 340-359 37 37 37 61 61 61 F F F
40942024-02-04T14:02:09.893Z 18 360-379 36 36 36 59 59 59 F F F
40952024-02-04T14:02:09.893Z 19 380-399 35 35 35 58 58 58 F F F
40962024-02-04T14:02:09.894Z 20 400-419 37 37 37 61 61 61 F F F
40972024-02-04T14:02:09.894Z 21 420-439 35 35 35 58 58 58 F F F
40982024-02-04T14:02:09.894Z 22 440-459 35 35 35 58 58 58 F F F
40992024-02-04T14:02:09.894Z 23 460-479 36 36 36 60 60 60 F F F
41002024-02-04T14:02:09.894Z 24 480-499 37 37 37 61 61 61 F F F
41012024-02-04T14:02:09.894Z 25 500-519 35 35 35 58 58 58 F F F
41022024-02-04T14:02:09.894Z 26 520-539 32 32 32 55 55 55 F F F
41032024-02-04T14:02:09.894Z 27 540-559 30 30 30 50 50 50 F F F
41042024-02-04T14:02:09.894Z 28 560-579 32 32 32 55 55 55 F F F
41052024-02-04T14:02:09.894Z 29 580-599 32 32 32 55 55 55 F F F
41062024-02-04T14:02:09.894ZMax gen: 37, Max flush: 61
41072024-02-04T14:02:09.897Z
41082024-02-04T14:02:09.897ZBegin loop 19
41092024-02-04T14:02:09.897ZDownstairs to restart: 1
41102024-02-04T14:02:09.927Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.959588685Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281}
41112024-02-04T14:02:09.927Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.960139576Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281}
41122024-02-04T14:02:09.927Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.960184498Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281}
41132024-02-04T14:02:09.927Z{"msg":"Crucible 36faef1a-daa8-4ab7-a72a-8c2300a47106 has session id: 796c0ab3-cb88-4b7d-863a-18517b4651d6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.960246718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41142024-02-04T14:02:09.931Z{"msg":"Upstairs opts: Upstairs UUID: 36faef1a-daa8-4ab7-a72a-8c2300a47106, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.960345472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41152024-02-04T14:02:09.931Z{"msg":"Crucible stats registered with UUID: 36faef1a-daa8-4ab7-a72a-8c2300a47106","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.960454396Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41162024-02-04T14:02:09.931ZCrucible runtime is spawned
41172024-02-04T14:02:09.931Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.96082669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281}
41182024-02-04T14:02:09.931Z{"msg":"36faef1a-daa8-4ab7-a72a-8c2300a47106 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961039781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41192024-02-04T14:02:09.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961102019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41202024-02-04T14:02:09.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961129362Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41212024-02-04T14:02:09.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961152475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41222024-02-04T14:02:09.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961223895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41232024-02-04T14:02:09.931Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961270721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41242024-02-04T14:02:09.931Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961351042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41252024-02-04T14:02:09.931Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961399328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"io task","client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41262024-02-04T14:02:09.931Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961479137Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"io task","client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41272024-02-04T14:02:09.931Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961556027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"io task","client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41282024-02-04T14:02:09.931Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.96178923Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"io task","client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41292024-02-04T14:02:09.932Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961840216Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"io task","client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41302024-02-04T14:02:09.932Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.961918927Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"io task","client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41312024-02-04T14:02:09.932Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.963034325Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41322024-02-04T14:02:09.932Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.963118899Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41332024-02-04T14:02:09.932Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.963151961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41342024-02-04T14:02:09.932Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.963193379Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41352024-02-04T14:02:09.932Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.963987544Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41362024-02-04T14:02:09.932Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.964205495Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41372024-02-04T14:02:09.936Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.964800224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41382024-02-04T14:02:09.936Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.964856657Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41392024-02-04T14:02:09.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.965082494Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41402024-02-04T14:02:09.936Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.965138182Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41412024-02-04T14:02:09.936Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.965163315Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41422024-02-04T14:02:09.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.965184693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41432024-02-04T14:02:09.936Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.965208114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41442024-02-04T14:02:09.936Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.965230453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41452024-02-04T14:02:09.936Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.965722402Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41462024-02-04T14:02:09.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.966509387Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41472024-02-04T14:02:09.937Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.96660033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41482024-02-04T14:02:09.937Z{"msg":"[0]R flush_numbers[0..12]: [58, 56, 60, 60, 58, 60, 43, 42, 61, 52, 61, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.966649886Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41492024-02-04T14:02:09.937Z{"msg":"[0]R generation[0..12]: [35, 33, 36, 36, 35, 36, 25, 24, 37, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.966721368Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41502024-02-04T14:02:09.937Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.966796162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41512024-02-04T14:02:09.937Z{"msg":"[1]R flush_numbers[0..12]: [58, 56, 60, 60, 58, 60, 43, 42, 61, 52, 61, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.966869944Z","Wait for a query_work_queue command to finish before sending IO
41522024-02-04T14:02:09.937Zhostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41532024-02-04T14:02:09.937Z{"msg":"[1]R generation[0..12]: [35, 33, 36, 36, 35, 36, 25, 24, 37, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.966924367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41542024-02-04T14:02:09.937Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967004603Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41552024-02-04T14:02:09.937Z{"msg":"[2]R flush_numbers[0..12]: [58, 56, 60, 60, 58, 60, 43, 42, 61, 52, 61, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967084114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41562024-02-04T14:02:09.937Z{"msg":"[2]R generation[0..12]: [35, 33, 36, 36, 35, 36, 25, 24, 37, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967180597Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41572024-02-04T14:02:09.937Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967260527Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41582024-02-04T14:02:09.937Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967306577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41592024-02-04T14:02:09.937Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967381301Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41602024-02-04T14:02:09.937Z{"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967427813Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41612024-02-04T14:02:09.937Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967455896Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41622024-02-04T14:02:09.937Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.96752725Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41632024-02-04T14:02:09.937Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41642024-02-04T14:02:09.937Z,"time":"2024-02-04T14:02:08.967596297Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41652024-02-04T14:02:09.937Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967668982Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41662024-02-04T14:02:09.937Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967696762Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41672024-02-04T14:02:09.937Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967738431Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41682024-02-04T14:02:09.937Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967761328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41692024-02-04T14:02:09.937Z{"msg":"36faef1a-daa8-4ab7-a72a-8c2300a47106 is now active with session: 796c0ab3-cb88-4b7d-863a-18517b4651d6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967783588Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41702024-02-04T14:02:09.937Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967822163Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281,"session_id":"796c0ab3-cb88-4b7d-863a-18517b4651d6"}
41712024-02-04T14:02:09.938Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:08.967844581Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281}
41722024-02-04T14:02:09.938ZLoading write count information from file "/tmp/test_repair_verify.data"
41732024-02-04T14:02:09.938ZRun Repair workload
41742024-02-04T14:02:09.938Z01/30 Write block 353 len 16384 data: 5 4 5 5
41752024-02-04T14:02:09.941Z02/30 Write block 213 len 36864 data: 4 5 6 7 7 5 4 4 4
41762024-02-04T14:02:09.941Z03/30 Read block 31 len 4096
41772024-02-04T14:02:09.941Z04/30 Read block 171 len 36864
41782024-02-04T14:02:09.944Z05/30 Read block 550 len 12288
41792024-02-04T14:02:09.944Z06/30 Read block 466 len 28672
41802024-02-04T14:02:09.947Z07/30 Write block 125 len 28672 data: 4 6 4 3 2 2 2
41812024-02-04T14:02:09.947Z08/30 Write block 501 len 4096 data: 5
41822024-02-04T14:02:09.947Z09/30 Write block 475 len 4096 data: 4
41832024-02-04T14:02:09.950Z10/30 Read block 424 len 36864
41842024-02-04T14:02:09.950Z11/30 Read block 166 len 16384
41852024-02-04T14:02:09.954Z12/30 Flush
41862024-02-04T14:02:09.957Z13/30 Write block 331 len 32768 data: 4 4 4 5 4 4 5 4
41872024-02-04T14:02:09.957Z14/30 Flush
41882024-02-04T14:02:09.960Z15/30 Flush
41892024-02-04T14:02:09.960Z16/30 Write block 330 len 8192 data: 5 5
41902024-02-04T14:02:09.960Z17/30 Read block 389 len 36864
41912024-02-04T14:02:09.963Z18/30 Write block 212 len 12288 data: 4 5 6
41922024-02-04T14:02:09.963Z19/30 Read block 137 len 16384
41932024-02-04T14:02:09.966Z20/30 Write block 471 len 16384 data: 4 5 5 5
41942024-02-04T14:02:09.966Z21/30 Read block 369 len 8192
41952024-02-04T14:02:09.968Z22/30 Write block 363 len 8192 data: 5 5
41962024-02-04T14:02:09.969Z23/30 Flush
41972024-02-04T14:02:09.972Z24/30 Write block 532 len 40960 data: 2 3 5 5 5 3 3 3 4 5
41982024-02-04T14:02:09.975Z25/30 Write block 584 len 20480 data: 7 6 5 5 4
41992024-02-04T14:02:09.975Z26/30 Read block 228 len 4096
42002024-02-04T14:02:09.977Z27/30 Read block 196 len 8192
42012024-02-04T14:02:09.978Z28/30 Read block 10 len 32768
42022024-02-04T14:02:09.978Z29/30 Write block 374 len 12288 data: 4 4 3
42032024-02-04T14:02:09.981Z30/30 Write block 24 len 24576 data: 2 2 2 2 3 3
42042024-02-04T14:02:09.981Z----------------------------------------------------------------
42052024-02-04T14:02:09.982Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
42062024-02-04T14:02:09.982ZStates: Active Active Active
42072024-02-04T14:02:09.982ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42082024-02-04T14:02:09.982Z 1 Acked 1000 Write 4 Done Done Done false
42092024-02-04T14:02:09.982Z 2 Acked 1001 Write 9 Done Sent Done false
42102024-02-04T14:02:09.982Z 3 Acked 1002 Read 1 Done Sent Done false
42112024-02-04T14:02:09.982Z 4 Acked 1003 Read 9 Done Sent Done false
42122024-02-04T14:02:09.982Z 5 Acked 1004 Read 3 Done Sent Done false
42132024-02-04T14:02:09.982Z 6 Acked 1005 Read 7 Done Sent Done false
42142024-02-04T14:02:09.982Z 7 Acked 1006 Write 7 Done Sent Done false
42152024-02-04T14:02:09.982Z 8 Acked 1007 Write 1 Done Sent Done false
42162024-02-04T14:02:09.982Z 9 Acked 1008 Write 1 Done Sent Done false
42172024-02-04T14:02:09.982Z 10 Acked 1009 Read 9 Done Sent Done false
42182024-02-04T14:02:09.982Z 11 Acked 1010 Read 4 Done Sent Done false
42192024-02-04T14:02:09.982Z 12 Acked 1011 Flush 0 Done Sent Done false
42202024-02-04T14:02:09.982Z 13 Acked 1012 Write 8 Done Sent Done false
42212024-02-04T14:02:09.982Z 14 Acked 1013 Flush 0 Done Sent Done false
42222024-02-04T14:02:09.982Z 15 Acked 1014 Flush 0 Done Sent Done false
42232024-02-04T14:02:09.982Z 16 Acked 1015 Write 2 Done Sent Done false
42242024-02-04T14:02:09.982Z 17 Acked 1016 Read 9 Done Sent Done false
42252024-02-04T14:02:09.982Z 18 Acked 1017 Write 3 Done Sent Done false
42262024-02-04T14:02:09.982Z 19 Acked 1018 Read 4 Done Sent Done false
42272024-02-04T14:02:09.982Z 20 Acked 1019 Write 4 Done Sent Done false
42282024-02-04T14:02:09.982Z 21 Acked 1020 Read 2 Done Sent Done false
42292024-02-04T14:02:09.983Z 22 Acked 1021 Write 2 Done Sent Done false
42302024-02-04T14:02:09.983Z 23 Acked 1022 Flush 0 Done Sent Done false
42312024-02-04T14:02:09.983Z 24 Acked 1023 Write 10 Done Sent Done false
42322024-02-04T14:02:09.983Z 25 Acked 1024 Write 5 Done Sent Done false
42332024-02-04T14:02:09.983Z 26 Acked 1025 Read 1 Done Sent Done false
42342024-02-04T14:02:09.983Z 27 Acked 1026 Read 2 Done Sent Done false
42352024-02-04T14:02:09.983Z 28 Acked 1027 Read 8 Done Sent Done false
42362024-02-04T14:02:09.983Z 29 Acked 1028 Write 3 Done Sent Done false
42372024-02-04T14:02:09.983Z 30 Acked 1029 Write 6 Sent Sent Sent false
42382024-02-04T14:02:09.983Z STATES DS:0 DS:1 DS:2 TOTAL
42392024-02-04T14:02:09.983Z New 0 0 0 0
42402024-02-04T14:02:09.983Z Sent 1 29 1 31
42412024-02-04T14:02:09.983Z Done 29 1 29 59
42422024-02-04T14:02:09.983Z Skipped 0 0 0 0
42432024-02-04T14:02:09.983Z Error 0 0 0 0
42442024-02-04T14:02:09.983ZLast Flush: 1022 0 1022
42452024-02-04T14:02:09.983ZDownstairs last five completed:
42462024-02-04T14:02:09.983ZUpstairs last five completed: 30 29 28 27 26
42472024-02-04T14:02:09.983Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:09.013996829Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1281}
42482024-02-04T14:02:09.983ZWrote out file "/tmp/test_repair_verify.data"
42492024-02-04T14:02:09.988Z
42502024-02-04T14:02:09.995ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42512024-02-04T14:02:10.026ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42522024-02-04T14:02:10.026Z 0 000-019 35 35 35 58 58 58 F F F
42532024-02-04T14:02:10.026Z 1 020-039 33 33 33 56 56 56 T F T <---
42542024-02-04T14:02:10.026Z 2 040-059 36 36 36 60 60 60 F F F
42552024-02-04T14:02:10.026Z 3 060-079 36 36 36 60 60 60 F F F
42562024-02-04T14:02:10.026Z 4 080-099 35 35 35 58 58 58 F F F
42572024-02-04T14:02:10.027Z 5 100-119 36 36 36 60 60 60 F F F
42582024-02-04T14:02:10.027Z 6 120-139 38 25 38 62 43 62 F F F <---
42592024-02-04T14:02:10.027Z 7 140-159 24 24 24 42 42 42 F F F
42602024-02-04T14:02:10.027Z 8 160-179 37 37 37 61 61 61 F F F
42612024-02-04T14:02:10.027Z 9 180-199 30 30 30 52 52 52 F F F
42622024-02-04T14:02:10.027Z 10 200-219 38 37 38 65 61 65 F F F <---
42632024-02-04T14:02:10.030Z 11 220-239 38 35 38 62 58 62 F F F <---
42642024-02-04T14:02:10.030Z 12 240-259 36 36 36 59 59 59 F F F
42652024-02-04T14:02:10.030Z 13 260-279 37 37 37 61 61 61 F F F
42662024-02-04T14:02:10.030Z 14 280-299 37 37 37 61 61 61 F F F
42672024-02-04T14:02:10.030Z 15 300-319 25 25 25 43 43 43 F F F
42682024-02-04T14:02:10.030Z 16 320-339 38 33 38 65 56 65 F F F <---
42692024-02-04T14:02:10.030Z 17 340-359 38 37 38 62 61 62 F T F <---
42702024-02-04T14:02:10.031Z 18 360-379 38 36 38 65 59 65 T F T <---
42712024-02-04T14:02:10.031Z 19 380-399 35 35 35 58 58 58 F F F
42722024-02-04T14:02:10.031Z 20 400-419 37 37 37 61 61 61 F F F
42732024-02-04T14:02:10.031Z 21 420-439 35 35 35 58 58 58 F F F
42742024-02-04T14:02:10.031Z 22 440-459 35 35 35 58 58 58 F F F
42752024-02-04T14:02:10.031Z 23 460-479 38 36 38 65 60 65 F F F <---
42762024-02-04T14:02:10.031Z 24 480-499 37 37 37 61 61 61 F F F
42772024-02-04T14:02:10.031Z 25 500-519 38 35 38 62 58 62 F F F <---
42782024-02-04T14:02:10.031Z 26 520-539 32 32 32 55 55 55 T F T <---
42792024-02-04T14:02:10.031Z 27 540-559 30 30 30 50 50 50 T F T <---
42802024-02-04T14:02:10.031Z 28 560-579 32 32 32 55 55 55 F F F
42812024-02-04T14:02:10.031Z 29 580-599 32 32 32 55 55 55 T F T <---
42822024-02-04T14:02:10.031ZMax gen: 38, Max flush: 65
42832024-02-04T14:02:10.031ZError: Difference in extent metadata found!
42842024-02-04T14:02:10.031ZA Difference in extent metadata is expected here
42852024-02-04T14:02:10.031ZOn loop 19
42862024-02-04T14:02:10.031Z
42872024-02-04T14:02:10.034ZVerifying data now
42882024-02-04T14:02:16.118ZNo change after verify
42892024-02-04T14:02:16.118ZLoop: 19 Downstairs dump after verify (and repair):
42902024-02-04T14:02:16.141ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42912024-02-04T14:02:16.142Z 0 000-019 35 35 35 58 58 58 F F F
42922024-02-04T14:02:16.142Z 1 020-039 39 39 39 66 66 66 F F F
42932024-02-04T14:02:16.142Z 2 040-059 36 36 36 60 60 60 F F F
42942024-02-04T14:02:16.142Z 3 060-079 36 36 36 60 60 60 F F F
42952024-02-04T14:02:16.142Z 4 080-099 35 35 35 58 58 58 F F F
42962024-02-04T14:02:16.146Z 5 100-119 36 36 36 60 60 60 F F F
42972024-02-04T14:02:16.146Z 6 120-139 38 38 38 62 62 62 F F F
42982024-02-04T14:02:16.146Z 7 140-159 24 24 24 42 42 42 F F F
42992024-02-04T14:02:16.146Z 8 160-179 37 37 37 61 61 61 F F F
43002024-02-04T14:02:16.146Z 9 180-199 30 30 30 52 52 52 F F F
43012024-02-04T14:02:16.146Z 10 200-219 38 38 38 65 65 65 F F F
43022024-02-04T14:02:16.146Z 11 220-239 38 38 38 62 62 62 F F F
43032024-02-04T14:02:16.146Z 12 240-259 36 36 36 59 59 59 F F F
43042024-02-04T14:02:16.146Z 13 260-279 37 37 37 61 61 61 F F F
43052024-02-04T14:02:16.146Z 14 280-299 37 37 37 61 61 61 F F F
43062024-02-04T14:02:16.146Z 15 300-319 25 25 25 43 43 43 F F F
43072024-02-04T14:02:16.146Z 16 320-339 38 38 38 65 65 65 F F F
43082024-02-04T14:02:16.146Z 17 340-359 38 38 38 62 62 62 F F F
43092024-02-04T14:02:16.146Z 18 360-379 39 39 39 66 66 66 F F F
43102024-02-04T14:02:16.147Z 19 380-399 35 35 35 58 58 58 F F F
43112024-02-04T14:02:16.147Z 20 400-419 37 37 37 61 61 61 F F F
43122024-02-04T14:02:16.147Z 21 420-439 35 35 35 58 58 58 F F F
43132024-02-04T14:02:16.147Z 22 440-459 35 35 35 58 58 58 F F F
43142024-02-04T14:02:16.147Z 23 460-479 38 38 38 65 65 65 F F F
43152024-02-04T14:02:16.147Z 24 480-499 37 37 37 61 61 61 F F F
43162024-02-04T14:02:16.147Z 25 500-519 38 38 38 62 62 62 F F F
43172024-02-04T14:02:16.147Z 26 520-539 39 39 39 66 66 66 F F F
43182024-02-04T14:02:16.147Z 27 540-559 39 39 39 66 66 66 F F F
43192024-02-04T14:02:16.147Z 28 560-579 32 32 32 55 55 55 F F F
43202024-02-04T14:02:16.147Z 29 580-599 39 39 39 66 66 66 F F F
43212024-02-04T14:02:16.147ZMax gen: 39, Max flush: 66
43222024-02-04T14:02:16.147Z
43232024-02-04T14:02:16.147ZBegin loop 20
43242024-02-04T14:02:16.147ZDownstairs to restart: 2
43252024-02-04T14:02:16.177Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.210067865Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289}
43262024-02-04T14:02:16.177Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.210559415Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289}
43272024-02-04T14:02:16.177Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.21059445Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289}
43282024-02-04T14:02:16.177Z{Crucible runtime is spawned
43292024-02-04T14:02:16.183Z"msg":"Crucible 2d1385de-0f81-4452-b47b-a86683b5671c has session id: bd3472a9-3516-4247-8442-0ebc3f3589a5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.210690009Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43302024-02-04T14:02:16.183Z{"msg":"Upstairs opts: Upstairs UUID: 2d1385de-0f81-4452-b47b-a86683b5671c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.210775759Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43312024-02-04T14:02:16.183Z{"msg":"Crucible stats registered with UUID: 2d1385de-0f81-4452-b47b-a86683b5671c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.210850035Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43322024-02-04T14:02:16.183Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.210877406Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289}
43332024-02-04T14:02:16.183Z{"msg":"2d1385de-0f81-4452-b47b-a86683b5671c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.21090451Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43342024-02-04T14:02:16.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.210941991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43352024-02-04T14:02:16.184Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.210983996Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43362024-02-04T14:02:16.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211103389Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43372024-02-04T14:02:16.184Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211176296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43382024-02-04T14:02:16.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211219027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43392024-02-04T14:02:16.184Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211538753Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43402024-02-04T14:02:16.184Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.21163573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43412024-02-04T14:02:16.184Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211668322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43422024-02-04T14:02:16.184Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211692498Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43432024-02-04T14:02:16.184Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211715622Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43442024-02-04T14:02:16.184Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211750646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43452024-02-04T14:02:16.184Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.211777833Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43462024-02-04T14:02:16.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.212691374Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43472024-02-04T14:02:16.184Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.212803805Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43482024-02-04T14:02:16.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.212929544Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43492024-02-04T14:02:16.184Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.212994935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43502024-02-04T14:02:16.184Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.213460303Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43512024-02-04T14:02:16.185Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.213518959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43522024-02-04T14:02:16.185Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.213637133Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43532024-02-04T14:02:16.185Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.213815484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43542024-02-04T14:02:16.185Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.214568677Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43552024-02-04T14:02:16.185Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.214693904Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43562024-02-04T14:02:16.185Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.214723093Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43572024-02-04T14:02:16.185Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.214766177Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43582024-02-04T14:02:16.185Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.214788523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43592024-02-04T14:02:16.185Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.214811283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43602024-02-04T14:02:16.185Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.214866123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43612024-02-04T14:02:16.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215356316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43622024-02-04T14:02:16.193Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215428581Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43632024-02-04T14:02:16.193Z{"msg":"[0]R flush_numbers[0..12]: [58, 66, 60, 60, 58, 60, 62, 42, 61, 52, 65, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215463405Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43642024-02-04T14:02:16.193Z{"msg":"[0]R generation[0..12]: [35, 39, 36, 36, 35, 36, 38, 24, 37, 30, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.21549118Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43652024-02-04T14:02:16.193Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215537261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43662024-02-04T14:02:16.193Z{"msg":"[1]R flush_numbers[0..12]: [58, 66, 60, 60, 58, 60, 62, 42, 61, 52, 65, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215562238Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43672024-02-04T14:02:16.194Z{"msg":"[1]R generation[0..12]: [35, 39, 36, 36, 35, 36, 38, 24, 37, 30, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215604227Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43682024-02-04T14:02:16.194Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215629716Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43692024-02-04T14:02:16.194Z{"msg":"[2]R flush_numbers[0..12]: [58, 66, 60, 60, 58, 60, 62, 42, 61, 52, 65, 62]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
43702024-02-04T14:02:16.194Z,"time":"2024-02-04T14:02:15.21567125Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43712024-02-04T14:02:16.194Z{"msg":"[2]R generation[0..12]: [35, 39, 36, 36, 35, 36, 38, 24, 37, 30, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215697083Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43722024-02-04T14:02:16.194Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215721035Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43732024-02-04T14:02:16.194Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215760758Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43742024-02-04T14:02:16.194Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215784431Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43752024-02-04T14:02:16.194Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215824386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43762024-02-04T14:02:16.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215847871Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43772024-02-04T14:02:16.194Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215888709Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43782024-02-04T14:02:16.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215910319Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43792024-02-04T14:02:16.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215967539Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43802024-02-04T14:02:16.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.215994706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43812024-02-04T14:02:16.194Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.216037063Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43822024-02-04T14:02:16.194Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.216059793Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43832024-02-04T14:02:16.194Z{"msg":"2d1385de-0f81-4452-b47b-a86683b5671c is now active with session: bd3472a9-3516-4247-8442-0ebc3f3589a5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.21608308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43842024-02-04T14:02:16.194Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.216103472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289,"session_id":"bd3472a9-3516-4247-8442-0ebc3f3589a5"}
43852024-02-04T14:02:16.194Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:15.216124129Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289}
43862024-02-04T14:02:16.194ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43872024-02-04T14:02:16.194ZLoading write count information from file "/tmp/test_repair_verify.data"
43882024-02-04T14:02:16.194ZRun Repair workload
43892024-02-04T14:02:16.194Z01/30 Write block 183 len 36864 data: 6 6 6 5 4 4 4 3 4
43902024-02-04T14:02:16.195Z02/30 Read block 573 len 4096
43912024-02-04T14:02:16.195Z03/30 Write block 574 len 16384 data: 4 4 4 4
43922024-02-04T14:02:16.198Z04/30 Read block 534 len 28672
43932024-02-04T14:02:16.198Z05/30 Read block 98 len 32768
43942024-02-04T14:02:16.198Z06/30 Read block 485 len 40960
43952024-02-04T14:02:16.198Z07/30 Read block 325 len 12288
43962024-02-04T14:02:16.198Z08/30 Read block 423 len 40960
43972024-02-04T14:02:16.198Z09/30 Read block 368 len 12288
43982024-02-04T14:02:16.202Z10/30 Write block 466 len 40960 data: 6 5 4 5 6 5 6 6 6 5
43992024-02-04T14:02:16.202Z11/30 Read block 276 len 20480
44002024-02-04T14:02:16.205Z12/30 Read block 310 len 40960
44012024-02-04T14:02:16.205Z13/30 Write block 387 len 12288 data: 2 2 3
44022024-02-04T14:02:16.208Z14/30 Read block 6 len 36864
44032024-02-04T14:02:16.208Z15/30 Write block 318 len 28672 data: 4 4 3 3 3 3 3
44042024-02-04T14:02:16.213Z16/30 Write block 583 len 16384 data: 7 8 7 6
44052024-02-04T14:02:16.213Z17/30 Write block 9 len 4096 data: 2
44062024-02-04T14:02:16.213Z18/30 Write block 311 len 40960 data: 4 3 3 3 2 3 4 5 5 4
44072024-02-04T14:02:16.217Z19/30 Write block 233 len 40960 data: 3 3 2 2 3 3 3 4 4 3
44082024-02-04T14:02:16.217Z20/30 Flush
44092024-02-04T14:02:16.220Z21/30 Write block 211 len 4096 data: 4
44102024-02-04T14:02:16.220Z22/30 Read block 215 len 32768
44112024-02-04T14:02:16.224Z23/30 Write block 284 len 8192 data: 6 6
44122024-02-04T14:02:16.224Z24/30 Write block 127 len 8192 data: 5 4
44132024-02-04T14:02:16.224Z25/30 Read block 98 len 12288
44142024-02-04T14:02:16.227Z26/30 Read block 236 len 24576
44152024-02-04T14:02:16.227Z27/30 Read block 160 len 32768
44162024-02-04T14:02:16.231Z28/30 Write block 383 len 36864 data: 2 2 2 2 3 3 4 3 5
44172024-02-04T14:02:16.231Z29/30 Read block 422 len 20480
44182024-02-04T14:02:16.234Z30/30 Write block 29 len 12288 data: 4 4 5
44192024-02-04T14:02:16.235Z----------------------------------------------------------------
44202024-02-04T14:02:16.235Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44212024-02-04T14:02:16.235ZStates: Active Active Active
44222024-02-04T14:02:16.235ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44232024-02-04T14:02:16.235Z 1 Acked 1000 Write 9 Done Done Done false
44242024-02-04T14:02:16.235Z 2 Acked 1001 Read 1 Done Done Done false
44252024-02-04T14:02:16.235Z 3 Acked 1002 Write 4 Done Done Done false
44262024-02-04T14:02:16.235Z 4 Acked 1003 Read 7 Done Done Sent false
44272024-02-04T14:02:16.235Z 5 Acked 1004 Read 8 Done Done Sent false
44282024-02-04T14:02:16.235Z 6 Acked 1005 Read 10 Done Done Sent false
44292024-02-04T14:02:16.235Z 7 Acked 1006 Read 3 Done Done Sent false
44302024-02-04T14:02:16.235Z 8 Acked 1007 Read 10 Done Done Sent false
44312024-02-04T14:02:16.235Z 9 Acked 1008 Read 3 Done Done Sent false
44322024-02-04T14:02:16.235Z 10 Acked 1009 Write 10 Done Done Sent false
44332024-02-04T14:02:16.235Z 11 Acked 1010 Read 5 Done Done Sent false
44342024-02-04T14:02:16.235Z 12 Acked 1011 Read 10 Done Done Sent false
44352024-02-04T14:02:16.235Z 13 Acked 1012 Write 3 Done Done Sent false
44362024-02-04T14:02:16.235Z 14 Acked 1013 Read 9 Done Done Sent false
44372024-02-04T14:02:16.235Z 15 Acked 1014 Write 7 Done Done Sent false
44382024-02-04T14:02:16.235Z 16 Acked 1015 Write 4 Done Done Sent false
44392024-02-04T14:02:16.235Z 17 Acked 1016 Write 1 Done Done Sent false
44402024-02-04T14:02:16.235Z 18 Acked 1017 Write 10 Done Done Sent false
44412024-02-04T14:02:16.235Z 19 Acked 1018 Write 10 Done Done Sent false
44422024-02-04T14:02:16.236Z 20 Acked 1019 Flush 0 Done Done Sent false
44432024-02-04T14:02:16.236Z 21 Acked 1020 Write 1 Done Done Sent false
44442024-02-04T14:02:16.236Z 22 Acked 1021 Read 8 Done Done Sent false
44452024-02-04T14:02:16.236Z 23 Acked 1022 Write 2 Done Done Sent false
44462024-02-04T14:02:16.236Z 24 Acked 1023 Write 2 Done Done Sent false
44472024-02-04T14:02:16.236Z 25 Acked 1024 Read 3 Done Done Sent false
44482024-02-04T14:02:16.236Z 26 Acked 1025 Read 6 Done Done Sent false
44492024-02-04T14:02:16.236Z 27 Acked 1026 Read 8 Done Done Sent false
44502024-02-04T14:02:16.236Z 28 Acked 1027 Write 9 Done Done Sent false
44512024-02-04T14:02:16.236Z 29 Acked 1028 Read 5 Done Done Sent false
44522024-02-04T14:02:16.236Z 30 Acked 1029 Write 3 Sent Sent Sent false
44532024-02-04T14:02:16.236Z STATES DS:0 DS:1 DS:2 TOTAL
44542024-02-04T14:02:16.236Z New 0 0 0 0
44552024-02-04T14:02:16.236Z Sent 1 1 27 29
44562024-02-04T14:02:16.236Z Done 29 29 3 61
44572024-02-04T14:02:16.236Z Skipped 0 0 0 0
44582024-02-04T14:02:16.236Z Error 0 0 0 0
44592024-02-04T14:02:16.236ZLast Flush: 1019 1019 0
44602024-02-04T14:02:16.236ZDownstairs last five completed:
44612024-02-04T14:02:16.236ZUpstairs last five completed: 30 29 28 27 26
44622024-02-04T14:02:16.236Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:15.266477887Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1289}
44632024-02-04T14:02:16.236ZWrote out file "/tmp/test_repair_verify.data"
44642024-02-04T14:02:16.240Z
44652024-02-04T14:02:16.246ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44662024-02-04T14:02:16.272ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44672024-02-04T14:02:16.272Z 0 000-019 40 40 35 67 67 58 F F F <---
44682024-02-04T14:02:16.272Z 1 020-039 39 39 39 66 66 66 T T F <---
44692024-02-04T14:02:16.273Z 2 040-059 36 36 36 60 60 60 F F F
44702024-02-04T14:02:16.273Z 3 060-079 36 36 36 60 60 60 F F F
44712024-02-04T14:02:16.273Z 4 080-099 35 35 35 58 58 58 F F F
44722024-02-04T14:02:16.273Z 5 100-119 36 36 36 60 60 60 F F F
44732024-02-04T14:02:16.273Z 6 120-139 38 38 38 62 62 62 T T F <---
44742024-02-04T14:02:16.273Z 7 140-159 24 24 24 42 42 42 F F F
44752024-02-04T14:02:16.273Z 8 160-179 37 37 37 61 61 61 F F F
44762024-02-04T14:02:16.273Z 9 180-199 40 40 30 67 67 52 F F T <---
44772024-02-04T14:02:16.273Z 10 200-219 38 38 38 65 65 65 T T F <---
44782024-02-04T14:02:16.276Z 11 220-239 40 40 38 67 67 62 F F F <---
44792024-02-04T14:02:16.277Z 12 240-259 40 40 36 67 67 59 F F F <---
44802024-02-04T14:02:16.277Z 13 260-279 37 37 37 61 61 61 F F F
44812024-02-04T14:02:16.277Z 14 280-299 37 37 37 61 61 61 T T F <---
44822024-02-04T14:02:16.277Z 15 300-319 40 40 25 67 67 43 F F F <---
44832024-02-04T14:02:16.277Z 16 320-339 40 40 38 67 67 65 F F F <---
44842024-02-04T14:02:16.277Z 17 340-359 38 38 38 62 62 62 F F F
44852024-02-04T14:02:16.277Z 18 360-379 39 39 39 66 66 66 F F F
44862024-02-04T14:02:16.277Z 19 380-399 40 40 35 67 67 58 T T F <---
44872024-02-04T14:02:16.277Z 20 400-419 37 37 37 61 61 61 F F F
44882024-02-04T14:02:16.277Z 21 420-439 35 35 35 58 58 58 F F F
44892024-02-04T14:02:16.277Z 22 440-459 35 35 35 58 58 58 F F F
44902024-02-04T14:02:16.277Z 23 460-479 40 40 38 67 67 65 F F F <---
44912024-02-04T14:02:16.277Z 24 480-499 37 37 37 61 61 61 F F F
44922024-02-04T14:02:16.277Z 25 500-519 38 38 38 62 62 62 F F F
44932024-02-04T14:02:16.277Z 26 520-539 39 39 39 66 66 66 F F F
44942024-02-04T14:02:16.278Z 27 540-559 39 39 39 66 66 66 F F F
44952024-02-04T14:02:16.278Z 28 560-579 40 40 32 67 67 55 F F T <---
44962024-02-04T14:02:16.278Z 29 580-599 40 40 39 67 67 66 F F F <---
44972024-02-04T14:02:16.278ZMax gen: 40, Max flush: 67
44982024-02-04T14:02:16.278ZError: Difference in extent metadata found!
44992024-02-04T14:02:16.294ZA Difference in extent metadata is expected here
45002024-02-04T14:02:16.294ZOn loop 20
45012024-02-04T14:02:16.294Z
45022024-02-04T14:02:16.299ZVerifying data now
45032024-02-04T14:02:22.510ZNo change after verify
45042024-02-04T14:02:22.510ZLoop: 20 Downstairs dump after verify (and repair):
45052024-02-04T14:02:22.533ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45062024-02-04T14:02:22.533Z 0 000-019 40 40 40 67 67 67 F F F
45072024-02-04T14:02:22.533Z 1 020-039 41 41 41 68 68 68 F F F
45082024-02-04T14:02:22.533Z 2 040-059 36 36 36 60 60 60 F F F
45092024-02-04T14:02:22.533Z 3 060-079 36 36 36 60 60 60 F F F
45102024-02-04T14:02:22.533Z 4 080-099 35 35 35 58 58 58 F F F
45112024-02-04T14:02:22.533Z 5 100-119 36 36 36 60 60 60 F F F
45122024-02-04T14:02:22.536Z 6 120-139 41 41 41 68 68 68 F F F
45132024-02-04T14:02:22.536Z 7 140-159 24 24 24 42 42 42 F F F
45142024-02-04T14:02:22.536Z 8 160-179 37 37 37 61 61 61 F F F
45152024-02-04T14:02:22.536Z 9 180-199 40 40 40 67 67 67 F F F
45162024-02-04T14:02:22.536Z 10 200-219 41 41 41 68 68 68 F F F
45172024-02-04T14:02:22.536Z 11 220-239 40 40 40 67 67 67 F F F
45182024-02-04T14:02:22.536Z 12 240-259 40 40 40 67 67 67 F F F
45192024-02-04T14:02:22.536Z 13 260-279 37 37 37 61 61 61 F F F
45202024-02-04T14:02:22.536Z 14 280-299 41 41 41 68 68 68 F F F
45212024-02-04T14:02:22.536Z 15 300-319 40 40 40 67 67 67 F F F
45222024-02-04T14:02:22.536Z 16 320-339 40 40 40 67 67 67 F F F
45232024-02-04T14:02:22.536Z 17 340-359 38 38 38 62 62 62 F F F
45242024-02-04T14:02:22.536Z 18 360-379 39 39 39 66 66 66 F F F
45252024-02-04T14:02:22.536Z 19 380-399 41 41 41 68 68 68 F F F
45262024-02-04T14:02:22.536Z 20 400-419 37 37 37 61 61 61 F F F
45272024-02-04T14:02:22.537Z 21 420-439 35 35 35 58 58 58 F F F
45282024-02-04T14:02:22.537Z 22 440-459 35 35 35 58 58 58 F F F
45292024-02-04T14:02:22.537Z 23 460-479 40 40 40 67 67 67 F F F
45302024-02-04T14:02:22.537Z 24 480-499 37 37 37 61 61 61 F F F
45312024-02-04T14:02:22.537Z 25 500-519 38 38 38 62 62 62 F F F
45322024-02-04T14:02:22.537Z 26 520-539 39 39 39 66 66 66 F F F
45332024-02-04T14:02:22.537Z 27 540-559 39 39 39 66 66 66 F F F
45342024-02-04T14:02:22.537Z 28 560-579 40 40 40 67 67 67 F F F
45352024-02-04T14:02:22.537Z 29 580-599 40 40 40 67 67 67 F F F
45362024-02-04T14:02:22.537ZMax gen: 41, Max flush: 68
45372024-02-04T14:02:22.540Z
45382024-02-04T14:02:22.540ZBegin loop 21
45392024-02-04T14:02:22.540ZDownstairs to restart: 0
45402024-02-04T14:02:22.567Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.600832916Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297}
45412024-02-04T14:02:22.567Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.60135798Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297}
45422024-02-04T14:02:22.571Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.601423458Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297}
45432024-02-04T14:02:22.571Z{"msg":"Crucible 102b1747-ebc1-4a5c-8b27-95e1362bde25 has session id: c1180336-e955-49d5-8249-ac75e28d5a27","v":0,"name":"crucible","level":30Crucible runtime is spawned
45442024-02-04T14:02:22.571Z,"time":"2024-02-04T14:02:21.601594009Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45452024-02-04T14:02:22.571Z{"msg":"Upstairs opts: Upstairs UUID: 102b1747-ebc1-4a5c-8b27-95e1362bde25, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.601655463Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45462024-02-04T14:02:22.571Z{"msg":"Crucible stats registered with UUID: 102b1747-ebc1-4a5c-8b27-95e1362bde25","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.601771522Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45472024-02-04T14:02:22.571Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.601814034Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297}
45482024-02-04T14:02:22.571Z{"msg":"102b1747-ebc1-4a5c-8b27-95e1362bde25 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.601877444Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45492024-02-04T14:02:22.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.601954165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45502024-02-04T14:02:22.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.601999001Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45512024-02-04T14:02:22.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602164878Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45522024-02-04T14:02:22.571Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602216553Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45532024-02-04T14:02:22.571Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602256771Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45542024-02-04T14:02:22.572Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602294881Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45552024-02-04T14:02:22.572Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.60233632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"io task","client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45562024-02-04T14:02:22.572Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602368348Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"io task","client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45572024-02-04T14:02:22.572Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602399883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"io task","client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45582024-02-04T14:02:22.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602514529Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"io task","client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45592024-02-04T14:02:22.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602648176Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"io task","client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45602024-02-04T14:02:22.572Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.602797697Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"io task","client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45612024-02-04T14:02:22.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.603893543Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45622024-02-04T14:02:22.572Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.604147446Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45632024-02-04T14:02:22.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.604199955Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45642024-02-04T14:02:22.572Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.604224934Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45652024-02-04T14:02:22.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.604247915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45662024-02-04T14:02:22.572Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.604270884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45672024-02-04T14:02:22.576Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.605052959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45682024-02-04T14:02:22.576Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.605165874Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45692024-02-04T14:02:22.576Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.605200778Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45702024-02-04T14:02:22.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.605889541Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45712024-02-04T14:02:22.576Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.605983247Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45722024-02-04T14:02:22.576Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606105076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45732024-02-04T14:02:22.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606166367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45742024-02-04T14:02:22.576Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606192684Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45752024-02-04T14:02:22.576Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606214853Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45762024-02-04T14:02:22.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606253944Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45772024-02-04T14:02:22.576Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606277934Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45782024-02-04T14:02:22.576Z{"msg":"[0]R flush_numbers[0..12]: [67, 68, 60, 60, 58, 60, 68, 42, 61, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606319615Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45792024-02-04T14:02:22.576Z{"msg":"[0]R generation[0..12]: [40, 41, 36, 36, 35, 36, 41, 24, 37, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606343904Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45802024-02-04T14:02:22.576Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606369813Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45812024-02-04T14:02:22.577Z{"msg":"[1]R flush_numbers[0..12]: [67, 68, 60, 60, 58, 60, 68, 42, 61, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO
45822024-02-04T14:02:22.577Z"2024-02-04T14:02:21.606393324Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45832024-02-04T14:02:22.577Z{"msg":"[1]R generation[0..12]: [40, 41, 36, 36, 35, 36, 41, 24, 37, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606442831Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45842024-02-04T14:02:22.577Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606467658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45852024-02-04T14:02:22.577Z{"msg":"[2]R flush_numbers[0..12]: [67, 68, 60, 60, 58, 60, 68, 42, 61, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606513163Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45862024-02-04T14:02:22.577Z{"msg":"[2]R generation[0..12]: [40, 41, 36, 36, 35, 36, 41, 24, 37, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606537048Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45872024-02-04T14:02:22.577Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606560863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45882024-02-04T14:02:22.577Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606584287Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45892024-02-04T14:02:22.577Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606606767Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45902024-02-04T14:02:22.577Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.60662944Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45912024-02-04T14:02:22.577Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606669897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45922024-02-04T14:02:22.577Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606693143Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45932024-02-04T14:02:22.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606713247Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45942024-02-04T14:02:22.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606753712Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45952024-02-04T14:02:22.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606776494Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45962024-02-04T14:02:22.577Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606798508Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45972024-02-04T14:02:22.577Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606819112Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45982024-02-04T14:02:22.577Z{"msg":"102b1747-ebc1-4a5c-8b27-95e1362bde25 is now active with session: c1180336-e955-49d5-8249-ac75e28d5a27","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.60687327Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
45992024-02-04T14:02:22.577Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606896926Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297,"session_id":"c1180336-e955-49d5-8249-ac75e28d5a27"}
46002024-02-04T14:02:22.577Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:21.606945838Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297}
46012024-02-04T14:02:22.577ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
46022024-02-04T14:02:22.577ZLoading write count information from file "/tmp/test_repair_verify.data"
46032024-02-04T14:02:22.577ZRun Repair workload
46042024-02-04T14:02:22.577Z01/30 Write block 120 len 8192 data: 3 3
46052024-02-04T14:02:22.581Z02/30 Read block 437 len 36864
46062024-02-04T14:02:22.581Z03/30 Read block 518 len 32768
46072024-02-04T14:02:22.581Z04/30 Read block 107 len 28672
46082024-02-04T14:02:22.584Z05/30 Read block 544 len 40960
46092024-02-04T14:02:22.584Z06/30 Read block 276 len 12288
46102024-02-04T14:02:22.587Z07/30 Write block 576 len 24576 data: 5 5 4 6 7 7
46112024-02-04T14:02:22.587Z08/30 Write block 388 len 4096 data: 4
46122024-02-04T14:02:22.592Z09/30 Write block 549 len 8192 data: 3 4
46132024-02-04T14:02:22.592Z10/30 Write block 579 len 40960 data: 7 8 8 6 8 9 8 7 6 5
46142024-02-04T14:02:22.593Z11/30 Flush
46152024-02-04T14:02:22.595Z12/30 Write block 375 len 40960 data: 5 4 4 5 5 6 4 3 3 3
46162024-02-04T14:02:22.598Z13/30 Write block 367 len 40960 data: 4 4 4 4 3 3 2 5 6 5
46172024-02-04T14:02:22.598Z14/30 Read block 157 len 24576
46182024-02-04T14:02:22.601Z15/30 Write block 172 len 32768 data: 9 9 9 8 7 7 6 7
46192024-02-04T14:02:22.603Z16/30 Write block 107 len 12288 data: 4 6 6
46202024-02-04T14:02:22.603Z17/30 Write block 543 len 24576 data: 5 3 3 2 2 2
46212024-02-04T14:02:22.606Z18/30 Write block 122 len 36864 data: 3 3 3 5 7 6 5 3 3
46222024-02-04T14:02:22.606Z19/30 Write block 42 len 28672 data: 3 3 3 4 4 3 3
46232024-02-04T14:02:22.606Z20/30 Write block 469 len 8192 data: 6 7
46242024-02-04T14:02:22.609Z21/30 Read block 82 len 36864
46252024-02-04T14:02:22.612Z22/30 Write block 159 len 40960 data: 4 3 4 4 4 4 4 4 4 5
46262024-02-04T14:02:22.612Z23/30 Write block 2 len 32768 data: 2 2 3 3 3 3 2 3
46272024-02-04T14:02:22.612Z24/30 Read block 59 len 36864
46282024-02-04T14:02:22.615Z25/30 Read block 309 len 4096
46292024-02-04T14:02:22.618Z26/30 Write block 413 len 36864 data: 7 7 7 4 4 3 4 6 6
46302024-02-04T14:02:22.618Z27/30 Write block 401 len 24576 data: 3 3 3 3 4 4
46312024-02-04T14:02:22.618Z28/30 Read block 17 len 4096
46322024-02-04T14:02:22.620Z29/30 Write block 122 len 16384 data: 4 4 4 6
46332024-02-04T14:02:22.620Z30/30 Read block 77 len 8192
46342024-02-04T14:02:22.624Z----------------------------------------------------------------
46352024-02-04T14:02:22.624Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46362024-02-04T14:02:22.624ZStates: Active Active Active
46372024-02-04T14:02:22.624ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46382024-02-04T14:02:22.624Z 1 Acked 1000 Write 2 Done Done Done false
46392024-02-04T14:02:22.624Z 2 Acked 1001 Read 9 Sent Done Done false
46402024-02-04T14:02:22.624Z 3 Acked 1002 Read 8 Sent Done Done false
46412024-02-04T14:02:22.624Z 4 Acked 1003 Read 7 Sent Done Done false
46422024-02-04T14:02:22.624Z 5 Acked 1004 Read 10 Sent Done Done false
46432024-02-04T14:02:22.624Z 6 Acked 1005 Read 3 Sent Done Done false
46442024-02-04T14:02:22.624Z 7 Acked 1006 Write 6 Sent Done Done false
46452024-02-04T14:02:22.624Z 8 Acked 1007 Write 1 Sent Done Done false
46462024-02-04T14:02:22.624Z 9 Acked 1008 Write 2 Sent Done Done false
46472024-02-04T14:02:22.624Z 10 Acked 1009 Write 10 Sent Done Done false
46482024-02-04T14:02:22.625Z 11 Acked 1010 Flush 0 Sent Done Done false
46492024-02-04T14:02:22.625Z 12 Acked 1011 Write 10 Sent Done Done false
46502024-02-04T14:02:22.625Z 13 Acked 1012 Write 10 Sent Done Done false
46512024-02-04T14:02:22.625Z 14 Acked 1013 Read 6 Sent Done Done false
46522024-02-04T14:02:22.625Z 15 Acked 1014 Write 8 Sent Done Done false
46532024-02-04T14:02:22.625Z 16 Acked 1015 Write 3 Sent Done Done false
46542024-02-04T14:02:22.625Z 17 Acked 1016 Write 6 Sent Done Done false
46552024-02-04T14:02:22.625Z 18 Acked 1017 Write 9 Sent Done Done false
46562024-02-04T14:02:22.625Z 19 Acked 1018 Write 7 Sent Done Done false
46572024-02-04T14:02:22.625Z 20 Acked 1019 Write 2 Sent Done Done false
46582024-02-04T14:02:22.625Z 21 Acked 1020 Read 9 Sent Done Done false
46592024-02-04T14:02:22.625Z 22 Acked 1021 Write 10 Sent Done Done false
46602024-02-04T14:02:22.625Z 23 Acked 1022 Write 8 Sent Done Done false
46612024-02-04T14:02:22.625Z 24 Acked 1023 Read 9 Sent Done Done false
46622024-02-04T14:02:22.625Z 25 Acked 1024 Read 1 Sent Done Done false
46632024-02-04T14:02:22.625Z 26 Acked 1025 Write 9 Sent Done Done false
46642024-02-04T14:02:22.625Z 27 Acked 1026 Write 6 Sent Done Done false
46652024-02-04T14:02:22.625Z 28 Acked 1027 Read 1 Sent Done Done false
46662024-02-04T14:02:22.625Z 29 Acked 1028 Write 4 Sent Done Done false
46672024-02-04T14:02:22.625Z 30 Acked 1029 Read 2 Sent Done Done false
46682024-02-04T14:02:22.625Z STATES DS:0 DS:1 DS:2 TOTAL
46692024-02-04T14:02:22.625Z New 0 0 0 0
46702024-02-04T14:02:22.625Z Sent 29 0 0 29
46712024-02-04T14:02:22.625Z Done 1 30 30 61
46722024-02-04T14:02:22.625Z Skipped 0 0 0 0
46732024-02-04T14:02:22.626Z Error 0 0 0 0
46742024-02-04T14:02:22.626ZLast Flush: 0 1010 1010
46752024-02-04T14:02:22.626ZDownstairs last five completed:
46762024-02-04T14:02:22.626ZUpstairs last five completed: 30 29 28 27 26
46772024-02-04T14:02:22.626Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:21.655873693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1297}
46782024-02-04T14:02:22.626ZWrote out file "/tmp/test_repair_verify.data"
46792024-02-04T14:02:22.629Z
46802024-02-04T14:02:22.633ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46812024-02-04T14:02:22.662ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46822024-02-04T14:02:22.662Z 0 000-019 40 40 40 67 67 67 F T T <---
46832024-02-04T14:02:22.662Z 1 020-039 41 41 41 68 68 68 F F F
46842024-02-04T14:02:22.662Z 2 040-059 36 36 36 60 60 60 F T T <---
46852024-02-04T14:02:22.662Z 3 060-079 36 36 36 60 60 60 F F F
46862024-02-04T14:02:22.662Z 4 080-099 35 35 35 58 58 58 F F F
46872024-02-04T14:02:22.665Z 5 100-119 36 36 36 60 60 60 F T T <---
46882024-02-04T14:02:22.665Z 6 120-139 41 42 42 68 69 69 T T T <---
46892024-02-04T14:02:22.665Z 7 140-159 24 24 24 42 42 42 F T T <---
46902024-02-04T14:02:22.665Z 8 160-179 37 37 37 61 61 61 F T T <---
46912024-02-04T14:02:22.665Z 9 180-199 40 40 40 67 67 67 F F F
46922024-02-04T14:02:22.665Z 10 200-219 41 41 41 68 68 68 F F F
46932024-02-04T14:02:22.665Z 11 220-239 40 40 40 67 67 67 F F F
46942024-02-04T14:02:22.665Z 12 240-259 40 40 40 67 67 67 F F F
46952024-02-04T14:02:22.666Z 13 260-279 37 37 37 61 61 61 F F F
46962024-02-04T14:02:22.666Z 14 280-299 41 41 41 68 68 68 F F F
46972024-02-04T14:02:22.666Z 15 300-319 40 40 40 67 67 67 F F F
46982024-02-04T14:02:22.666Z 16 320-339 40 40 40 67 67 67 F F F
46992024-02-04T14:02:22.666Z 17 340-359 38 38 38 62 62 62 F F F
47002024-02-04T14:02:22.666Z 18 360-379 39 39 39 66 66 66 F T T <---
47012024-02-04T14:02:22.666Z 19 380-399 41 42 42 68 69 69 F T T <---
47022024-02-04T14:02:22.666Z 20 400-419 37 37 37 61 61 61 F T T <---
47032024-02-04T14:02:22.666Z 21 420-439 35 35 35 58 58 58 F T T <---
47042024-02-04T14:02:22.666Z 22 440-459 35 35 35 58 58 58 F F F
47052024-02-04T14:02:22.666Z 23 460-479 40 40 40 67 67 67 F T T <---
47062024-02-04T14:02:22.666Z 24 480-499 37 37 37 61 61 61 F F F
47072024-02-04T14:02:22.666Z 25 500-519 38 38 38 62 62 62 F F F
47082024-02-04T14:02:22.666Z 26 520-539 39 39 39 66 66 66 F F F
47092024-02-04T14:02:22.666Z 27 540-559 39 42 42 66 69 69 F T T <---
47102024-02-04T14:02:22.666Z 28 560-579 40 42 42 67 69 69 F F F <---
47112024-02-04T14:02:22.666Z 29 580-599 40 42 42 67 69 69 F F F <---
47122024-02-04T14:02:22.666ZMax gen: 42, Max flush: 69
47132024-02-04T14:02:22.666ZError: Difference in extent metadata found!
47142024-02-04T14:02:22.669ZA Difference in extent metadata is expected here
47152024-02-04T14:02:22.669ZOn loop 21
47162024-02-04T14:02:22.669Z
47172024-02-04T14:02:22.683ZVerifying data now
47182024-02-04T14:02:27.880ZNo change after verify
47192024-02-04T14:02:27.880ZLoop: 21 Downstairs dump after verify (and repair):
47202024-02-04T14:02:27.902ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47212024-02-04T14:02:27.902Z 0 000-019 43 43 43 70 70 70 F F F
47222024-02-04T14:02:27.902Z 1 020-039 41 41 41 68 68 68 F F F
47232024-02-04T14:02:27.902Z 2 040-059 43 43 43 70 70 70 F F F
47242024-02-04T14:02:27.902Z 3 060-079 36 36 36 60 60 60 F F F
47252024-02-04T14:02:27.902Z 4 080-099 35 35 35 58 58 58 F F F
47262024-02-04T14:02:27.903Z 5 100-119 43 43 43 70 70 70 F F F
47272024-02-04T14:02:27.903Z 6 120-139 43 43 43 70 70 70 F F F
47282024-02-04T14:02:27.906Z 7 140-159 43 43 43 70 70 70 F F F
47292024-02-04T14:02:27.906Z 8 160-179 43 43 43 70 70 70 F F F
47302024-02-04T14:02:27.906Z 9 180-199 40 40 40 67 67 67 F F F
47312024-02-04T14:02:27.906Z 10 200-219 41 41 41 68 68 68 F F F
47322024-02-04T14:02:27.906Z 11 220-239 40 40 40 67 67 67 F F F
47332024-02-04T14:02:27.906Z 12 240-259 40 40 40 67 67 67 F F F
47342024-02-04T14:02:27.906Z 13 260-279 37 37 37 61 61 61 F F F
47352024-02-04T14:02:27.906Z 14 280-299 41 41 41 68 68 68 F F F
47362024-02-04T14:02:27.906Z 15 300-319 40 40 40 67 67 67 F F F
47372024-02-04T14:02:27.906Z 16 320-339 40 40 40 67 67 67 F F F
47382024-02-04T14:02:27.906Z 17 340-359 38 38 38 62 62 62 F F F
47392024-02-04T14:02:27.906Z 18 360-379 43 43 43 70 70 70 F F F
47402024-02-04T14:02:27.906Z 19 380-399 43 43 43 70 70 70 F F F
47412024-02-04T14:02:27.906Z 20 400-419 43 43 43 70 70 70 F F F
47422024-02-04T14:02:27.906Z 21 420-439 43 43 43 70 70 70 F F F
47432024-02-04T14:02:27.906Z 22 440-459 35 35 35 58 58 58 F F F
47442024-02-04T14:02:27.907Z 23 460-479 43 43 43 70 70 70 F F F
47452024-02-04T14:02:27.907Z 24 480-499 37 37 37 61 61 61 F F F
47462024-02-04T14:02:27.907Z 25 500-519 38 38 38 62 62 62 F F F
47472024-02-04T14:02:27.907Z 26 520-539 39 39 39 66 66 66 F F F
47482024-02-04T14:02:27.907Z 27 540-559 43 43 43 70 70 70 F F F
47492024-02-04T14:02:27.907Z 28 560-579 42 42 42 69 69 69 F F F
47502024-02-04T14:02:27.907Z 29 580-599 42 42 42 69 69 69 F F F
47512024-02-04T14:02:27.907ZMax gen: 43, Max flush: 70
47522024-02-04T14:02:27.910Z
47532024-02-04T14:02:27.910ZBegin loop 22
47542024-02-04T14:02:27.910ZDownstairs to restart: 2
47552024-02-04T14:02:27.937Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.970055195Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305}
47562024-02-04T14:02:27.937Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.970519027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305}
47572024-02-04T14:02:27.940Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.970593737Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305}
47582024-02-04T14:02:27.940Z{"msg":"Crucible 4cbebd09-8e6f-45f8-8cb7-bc37002a8601 has session id: 0ae38e11-2de8-4650-8356-5206bf451f48","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.970752391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47592024-02-04T14:02:27.940Z{"msg":"Upstairs opts: Upstairs UUID: 4cbebd09-8e6f-45f8-8cb7-bc37002a8601, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.970805577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47602024-02-04T14:02:27.940Z{"msg":"Crucible stats registered with UUID: 4cbebd09-8e6f-45f8-8cb7-bc37002a8601","v":Crucible runtime is spawned
47612024-02-04T14:02:27.940Z0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.970923531Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47622024-02-04T14:02:27.941Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971006712Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305}
47632024-02-04T14:02:27.941Z{"msg":"4cbebd09-8e6f-45f8-8cb7-bc37002a8601 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971205989Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47642024-02-04T14:02:27.941Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971280496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47652024-02-04T14:02:27.941Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971328067Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47662024-02-04T14:02:27.941Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971369861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47672024-02-04T14:02:27.941Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971438107Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47682024-02-04T14:02:27.941Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971613569Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47692024-02-04T14:02:27.941Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971816583Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47702024-02-04T14:02:27.941Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.971921558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"io task","client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47712024-02-04T14:02:27.941Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.972019412Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"io task","client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47722024-02-04T14:02:27.941Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.972074464Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"io task","client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47732024-02-04T14:02:27.941Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.97209998Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"io task","client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47742024-02-04T14:02:27.941Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.972179985Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"io task","client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47752024-02-04T14:02:27.941Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.972221134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"io task","client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47762024-02-04T14:02:27.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.97300945Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47772024-02-04T14:02:27.941Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.973173241Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47782024-02-04T14:02:27.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.973274306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47792024-02-04T14:02:27.941Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.973305532Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47802024-02-04T14:02:27.941Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.973344492Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47812024-02-04T14:02:27.941Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.97339077Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47822024-02-04T14:02:27.945Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.974128773Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47832024-02-04T14:02:27.945Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.974219004Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47842024-02-04T14:02:27.945Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.974575069Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47852024-02-04T14:02:27.945Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.974966066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47862024-02-04T14:02:27.946Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975081868Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47872024-02-04T14:02:27.946Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975110518Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47882024-02-04T14:02:27.946Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975131554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47892024-02-04T14:02:27.946Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975180478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47902024-02-04T14:02:27.946Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975204395Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47912024-02-04T14:02:27.946Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975340632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47922024-02-04T14:02:27.946Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975374803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47932024-02-04T14:02:27.946Z{"msg":"[0]R flush_numbers[0..12]: [70, 68, 70, 60, 58, 70, 70, 70, 70, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975400635Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47942024-02-04T14:02:27.946Z{"msg":"[0]R generation[0..12]: [43, 41, 43, 36, 35, 43, 43, 43, 43, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975424576Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47952024-02-04T14:02:27.946Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975469723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47962024-02-04T14:02:27.946Z{"msg":"[1]R flush_numbers[0..12]: [70, 68, 70, 60, 58, 70, 70, 70, 70, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975494852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47972024-02-04T14:02:27.946Z{"msg":"[1]R generation[0..12]: [43, 41, 43, 36, 35, 43, 43, 43, 43, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.97551796Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47982024-02-04T14:02:27.946Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975542122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
47992024-02-04T14:02:27.946Z{"msg":"[2]R flush_numbers[0..12]: [70, 68, 70, 60, 58, 70, 70, 70, 70, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975565713Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48002024-02-04T14:02:27.946Z{"msg":"[2]R generation[0..12]: [43, 41, 43, 36, 35, 43, 43, 43, 43, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975588468Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48012024-02-04T14:02:27.946Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975611956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48022024-02-04T14:02:27.946Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975655273Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48032024-02-04T14:02:27.946Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
48042024-02-04T14:02:27.946Z,"time":"2024-02-04T14:02:26.975679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48052024-02-04T14:02:27.946Z{"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975705651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48062024-02-04T14:02:27.946Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975756591Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48072024-02-04T14:02:27.946Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.97578Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48082024-02-04T14:02:27.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975800837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48092024-02-04T14:02:27.946Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975851692Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48102024-02-04T14:02:27.947Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975874651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48112024-02-04T14:02:27.947Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975896629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48122024-02-04T14:02:27.947Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975917194Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48132024-02-04T14:02:27.947Z{"msg":"4cbebd09-8e6f-45f8-8cb7-bc37002a8601 is now active with session: 0ae38e11-2de8-4650-8356-5206bf451f48","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975964386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48142024-02-04T14:02:27.947Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.975987484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305,"session_id":"0ae38e11-2de8-4650-8356-5206bf451f48"}
48152024-02-04T14:02:27.947Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:26.97601819Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305}
48162024-02-04T14:02:27.947ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48172024-02-04T14:02:27.947ZLoading write count information from file "/tmp/test_repair_verify.data"
48182024-02-04T14:02:27.947ZRun Repair workload
48192024-02-04T14:02:27.947Z01/30 Write block 515 len 32768 data: 6 7 5 5 5 3 3 3
48202024-02-04T14:02:27.953Z02/30 Read block 24 len 40960
48212024-02-04T14:02:27.953Z03/30 Write block 564 len 20480 data: 6 5 5 4 3
48222024-02-04T14:02:27.953Z04/30 Read block 494 len 36864
48232024-02-04T14:02:27.956Z05/30 Read block 186 len 24576
48242024-02-04T14:02:27.956Z06/30 Write block 394 len 16384 data: 5 3 3 4
48252024-02-04T14:02:27.956Z07/30 Read block 429 len 28672
48262024-02-04T14:02:27.959Z08/30 Write block 248 len 40960 data: 4 6 7 6 6 3 3 4 3 3
48272024-02-04T14:02:27.959Z09/30 Write block 281 len 32768 data: 5 6 6 7 7 5 6 7
48282024-02-04T14:02:27.959Z10/30 Write block 569 len 24576 data: 4 3 3 4 4 5
48292024-02-04T14:02:27.962Z11/30 Read block 529 len 8192
48302024-02-04T14:02:27.962Z12/30 Write block 465 len 20480 data: 5 7 6 5 7
48312024-02-04T14:02:27.965Z13/30 Flush
48322024-02-04T14:02:27.968Z14/30 Write block 108 len 4096 data: 7
48332024-02-04T14:02:27.968Z15/30 Write block 475 len 28672 data: 6 4 4 4 4 4 3
48342024-02-04T14:02:27.971Z16/30 Write block 112 len 40960 data: 8 8 7 7 7 7 4 4 4 4
48352024-02-04T14:02:27.979Z17/30 Write block 252 len 36864 data: 7 4 4 5 4 4 4 4 3
48362024-02-04T14:02:27.979Z18/30 Write block 247 len 32768 data: 4 5 7 8 7 8 5 5
48372024-02-04T14:02:27.979Z19/30 Read block 539 len 20480
48382024-02-04T14:02:27.986Z20/30 Flush
48392024-02-04T14:02:27.986Z21/30 Write block 75 len 12288 data: 3 3 5
48402024-02-04T14:02:27.990Z22/30 Read block 168 len 20480
48412024-02-04T14:02:27.990Z23/30 Read block 365 len 16384
48422024-02-04T14:02:27.990Z24/30 Write block 311 len 12288 data: 5 4 4
48432024-02-04T14:02:27.990Z25/30 Read block 414 len 4096
48442024-02-04T14:02:27.990Z26/30 Read block 10 len 8192
48452024-02-04T14:02:27.990Z27/30 Read block 60 len 16384
48462024-02-04T14:02:27.990Z28/30 Write block 299 len 4096 data: 4
48472024-02-04T14:02:27.990Z29/30 Read block 284 len 8192
48482024-02-04T14:02:27.994Z30/30 Write block 36 len 28672 data: 6 6 4 5 3 2 4
48492024-02-04T14:02:27.994Z----------------------------------------------------------------
48502024-02-04T14:02:27.994Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48512024-02-04T14:02:27.994ZStates: Active Active Active
48522024-02-04T14:02:27.994ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48532024-02-04T14:02:27.994Z 1 Acked 1000 Write 8 Done Done Done false
48542024-02-04T14:02:27.994Z 2 Acked 1001 Read 10 Done Done Done false
48552024-02-04T14:02:27.994Z 3 Acked 1002 Write 5 Done Done Sent false
48562024-02-04T14:02:27.994Z 4 Acked 1003 Read 9 Done Done Sent false
48572024-02-04T14:02:27.994Z 5 Acked 1004 Read 6 Done Done Sent false
48582024-02-04T14:02:27.994Z 6 Acked 1005 Write 4 Done Done Sent false
48592024-02-04T14:02:27.994Z 7 Acked 1006 Read 7 Done Done Sent false
48602024-02-04T14:02:27.994Z 8 Acked 1007 Write 10 Done Done Sent false
48612024-02-04T14:02:27.994Z 9 Acked 1008 Write 8 Done Done Sent false
48622024-02-04T14:02:27.995Z 10 Acked 1009 Write 6 Done Done Sent false
48632024-02-04T14:02:27.995Z 11 Acked 1010 Read 2 Done Done Sent false
48642024-02-04T14:02:27.995Z 12 Acked 1011 Write 5 Done Done Sent false
48652024-02-04T14:02:27.995Z 13 Acked 1012 Flush 0 Done Done Sent false
48662024-02-04T14:02:27.995Z 14 Acked 1013 Write 1 Done Done Sent false
48672024-02-04T14:02:27.995Z 15 Acked 1014 Write 7 Done Done Sent false
48682024-02-04T14:02:27.995Z 16 Acked 1015 Write 10 Done Done Sent false
48692024-02-04T14:02:27.995Z 17 Acked 1016 Write 9 Done Done Sent false
48702024-02-04T14:02:27.995Z 18 Acked 1017 Write 8 Done Done Sent false
48712024-02-04T14:02:27.995Z 19 Acked 1018 Read 5 Done Done Sent false
48722024-02-04T14:02:27.995Z 20 Acked 1019 Flush 0 Done Done Sent false
48732024-02-04T14:02:27.995Z 21 Acked 1020 Write 3 Done Done Sent false
48742024-02-04T14:02:27.995Z 22 Acked 1021 Read 5 Done Done Sent false
48752024-02-04T14:02:27.995Z 23 Acked 1022 Read 4 Done Done Sent false
48762024-02-04T14:02:27.995Z 24 Acked 1023 Write 3 Done Done Sent false
48772024-02-04T14:02:27.995Z 25 Acked 1024 Read 1 Done Done Sent false
48782024-02-04T14:02:27.995Z 26 Acked 1025 Read 2 Done Done Sent false
48792024-02-04T14:02:27.995Z 27 Acked 1026 Read 4 Done Done Sent false
48802024-02-04T14:02:27.995Z 28 Acked 1027 Write 1 Done Done Sent false
48812024-02-04T14:02:27.995Z 29 Acked 1028 Read 2 Done Done Sent false
48822024-02-04T14:02:27.995Z 30 Acked 1029 Write 7 Sent Sent Sent false
48832024-02-04T14:02:27.995Z STATES DS:0 DS:1 DS:2 TOTAL
48842024-02-04T14:02:27.995Z New 0 0 0 0
48852024-02-04T14:02:27.996Z Sent 1 1 28 30
48862024-02-04T14:02:27.996Z Done 29 29 2 60
48872024-02-04T14:02:27.996Z Skipped 0 0 0 0
48882024-02-04T14:02:27.996Z Error 0 0 0 0
48892024-02-04T14:02:27.996ZLast Flush: 1019 1019 0
48902024-02-04T14:02:27.996ZDownstairs last five completed:
48912024-02-04T14:02:27.996ZUpstairs last five completed: 30 29 28 27 26
48922024-02-04T14:02:27.996Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:27.025764351Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1305}
48932024-02-04T14:02:27.996ZWrote out file "/tmp/test_repair_verify.data"
48942024-02-04T14:02:28.002Z
48952024-02-04T14:02:28.006ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48962024-02-04T14:02:28.027ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48972024-02-04T14:02:28.027Z 0 000-019 43 43 43 70 70 70 F F F
48982024-02-04T14:02:28.027Z 1 020-039 41 41 41 68 68 68 T T F <---
48992024-02-04T14:02:28.027Z 2 040-059 43 43 43 70 70 70 T T F <---
49002024-02-04T14:02:28.027Z 3 060-079 36 36 36 60 60 60 T T F <---
49012024-02-04T14:02:28.030Z 4 080-099 35 35 35 58 58 58 F F F
49022024-02-04T14:02:28.030Z 5 100-119 44 44 43 72 72 70 F F F <---
49032024-02-04T14:02:28.030Z 6 120-139 44 44 43 72 72 70 F F F <---
49042024-02-04T14:02:28.030Z 7 140-159 43 43 43 70 70 70 F F F
49052024-02-04T14:02:28.030Z 8 160-179 43 43 43 70 70 70 F F F
49062024-02-04T14:02:28.030Z 9 180-199 40 40 40 67 67 67 F F F
49072024-02-04T14:02:28.030Z 10 200-219 41 41 41 68 68 68 F F F
49082024-02-04T14:02:28.030Z 11 220-239 40 40 40 67 67 67 F F F
49092024-02-04T14:02:28.030Z 12 240-259 44 44 40 72 72 67 F F F <---
49102024-02-04T14:02:28.030Z 13 260-279 44 44 37 72 72 61 F F F <---
49112024-02-04T14:02:28.030Z 14 280-299 44 44 41 71 71 68 T T F <---
49122024-02-04T14:02:28.030Z 15 300-319 40 40 40 67 67 67 T T F <---
49132024-02-04T14:02:28.030Z 16 320-339 40 40 40 67 67 67 F F F
49142024-02-04T14:02:28.030Z 17 340-359 38 38 38 62 62 62 F F F
49152024-02-04T14:02:28.031Z 18 360-379 43 43 43 70 70 70 F F F
49162024-02-04T14:02:28.031Z 19 380-399 44 44 43 71 71 70 F F F <---
49172024-02-04T14:02:28.031Z 20 400-419 43 43 43 70 70 70 F F F
49182024-02-04T14:02:28.031Z 21 420-439 43 43 43 70 70 70 F F F
49192024-02-04T14:02:28.031Z 22 440-459 35 35 35 58 58 58 F F F
49202024-02-04T14:02:28.031Z 23 460-479 44 44 43 72 72 70 F F F <---
49212024-02-04T14:02:28.031Z 24 480-499 44 44 37 72 72 61 F F F <---
49222024-02-04T14:02:28.031Z 25 500-519 44 44 38 71 71 62 F F T <---
49232024-02-04T14:02:28.031Z 26 520-539 44 44 39 71 71 66 F F T <---
49242024-02-04T14:02:28.031Z 27 540-559 43 43 43 70 70 70 F F F
49252024-02-04T14:02:28.031Z 28 560-579 44 44 42 71 71 69 F F F <---
49262024-02-04T14:02:28.031Z 29 580-599 42 42 42 69 69 69 F F F
49272024-02-04T14:02:28.031ZMax gen: 44, Max flush: 72
49282024-02-04T14:02:28.031ZError: Difference in extent metadata found!
49292024-02-04T14:02:28.034ZA Difference in extent metadata is expected here
49302024-02-04T14:02:28.034ZOn loop 22
49312024-02-04T14:02:28.034Z
49322024-02-04T14:02:28.037ZVerifying data now
49332024-02-04T14:02:34.314ZNo change after verify
49342024-02-04T14:02:34.314ZLoop: 22 Downstairs dump after verify (and repair):
49352024-02-04T14:02:34.338ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49362024-02-04T14:02:34.338Z 0 000-019 43 43 43 70 70 70 F F F
49372024-02-04T14:02:34.338Z 1 020-039 45 45 45 73 73 73 F F F
49382024-02-04T14:02:34.338Z 2 040-059 45 45 45 73 73 73 F F F
49392024-02-04T14:02:34.338Z 3 060-079 45 45 45 73 73 73 F F F
49402024-02-04T14:02:34.342Z 4 080-099 35 35 35 58 58 58 F F F
49412024-02-04T14:02:34.342Z 5 100-119 44 44 44 72 72 72 F F F
49422024-02-04T14:02:34.342Z 6 120-139 44 44 44 72 72 72 F F F
49432024-02-04T14:02:34.342Z 7 140-159 43 43 43 70 70 70 F F F
49442024-02-04T14:02:34.342Z 8 160-179 43 43 43 70 70 70 F F F
49452024-02-04T14:02:34.342Z 9 180-199 40 40 40 67 67 67 F F F
49462024-02-04T14:02:34.342Z 10 200-219 41 41 41 68 68 68 F F F
49472024-02-04T14:02:34.342Z 11 220-239 40 40 40 67 67 67 F F F
49482024-02-04T14:02:34.342Z 12 240-259 44 44 44 72 72 72 F F F
49492024-02-04T14:02:34.342Z 13 260-279 44 44 44 72 72 72 F F F
49502024-02-04T14:02:34.342Z 14 280-299 45 45 45 73 73 73 F F F
49512024-02-04T14:02:34.342Z 15 300-319 45 45 45 73 73 73 F F F
49522024-02-04T14:02:34.342Z 16 320-339 40 40 40 67 67 67 F F F
49532024-02-04T14:02:34.342Z 17 340-359 38 38 38 62 62 62 F F F
49542024-02-04T14:02:34.342Z 18 360-379 43 43 43 70 70 70 F F F
49552024-02-04T14:02:34.342Z 19 380-399 44 44 44 71 71 71 F F F
49562024-02-04T14:02:34.342Z 20 400-419 43 43 43 70 70 70 F F F
49572024-02-04T14:02:34.342Z 21 420-439 43 43 43 70 70 70 F F F
49582024-02-04T14:02:34.343Z 22 440-459 35 35 35 58 58 58 F F F
49592024-02-04T14:02:34.343Z 23 460-479 44 44 44 72 72 72 F F F
49602024-02-04T14:02:34.343Z 24 480-499 44 44 44 72 72 72 F F F
49612024-02-04T14:02:34.343Z 25 500-519 44 44 44 71 71 71 F F F
49622024-02-04T14:02:34.343Z 26 520-539 44 44 44 71 71 71 F F F
49632024-02-04T14:02:34.343Z 27 540-559 43 43 43 70 70 70 F F F
49642024-02-04T14:02:34.343Z 28 560-579 44 44 44 71 71 71 F F F
49652024-02-04T14:02:34.343Z 29 580-599 42 42 42 69 69 69 F F F
49662024-02-04T14:02:34.343ZMax gen: 45, Max flush: 73
49672024-02-04T14:02:34.343Z
49682024-02-04T14:02:34.343ZBegin loop 23
49692024-02-04T14:02:34.343ZDownstairs to restart: 1
49702024-02-04T14:02:34.371Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.404620877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313}
49712024-02-04T14:02:34.371Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405112541Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313}
49722024-02-04T14:02:34.375Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405187556Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313}
49732024-02-04T14:02:34.375Z{"msg":"Crucible 9a08d979-545d-400b-964d-43bcd29ff99d has session id: f56a69d3-7a1d-46e6-a9f8-1447eea9812d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.40526021Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49742024-02-04T14:02:34.375Z{"msg":"Upstairs opts: Upstairs UUID: 9a08d979-545d-400b-964d-43bcd29ff99d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405338428Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49752024-02-04T14:02:34.375Z{"msg":"Crucible stats registered with UUID: 9a08d979-545d-400b-964d-43bcd29ff99d","v"Crucible runtime is spawned
49762024-02-04T14:02:34.375Z:0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405392355Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49772024-02-04T14:02:34.375Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405464283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313}
49782024-02-04T14:02:34.375Z{"msg":"9a08d979-545d-400b-964d-43bcd29ff99d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405693702Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49792024-02-04T14:02:34.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405781642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49802024-02-04T14:02:34.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405947441Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49812024-02-04T14:02:34.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.405993078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49822024-02-04T14:02:34.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.406113133Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49832024-02-04T14:02:34.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.406319707Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49842024-02-04T14:02:34.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.406692091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49852024-02-04T14:02:34.376Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.40673917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"io task","client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49862024-02-04T14:02:34.376Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.406777424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"io task","client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49872024-02-04T14:02:34.376Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.406864808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"io task","client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49882024-02-04T14:02:34.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.406949742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"io task","client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49892024-02-04T14:02:34.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.40710563Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"io task","client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49902024-02-04T14:02:34.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.407182904Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"io task","client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49912024-02-04T14:02:34.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.407527869Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49922024-02-04T14:02:34.376Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.407606629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49932024-02-04T14:02:34.376Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.407656989Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49942024-02-04T14:02:34.376Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.407735306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49952024-02-04T14:02:34.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.407783085Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49962024-02-04T14:02:34.377Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.407856623Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49972024-02-04T14:02:34.384Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.408647425Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49982024-02-04T14:02:34.384Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.408717429Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
49992024-02-04T14:02:34.384Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.408841365Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50002024-02-04T14:02:34.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.40984058Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50012024-02-04T14:02:34.384Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.409918977Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50022024-02-04T14:02:34.384Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410004149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50032024-02-04T14:02:34.384Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410057994Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50042024-02-04T14:02:34.385Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410082686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50052024-02-04T14:02:34.385Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410129408Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50062024-02-04T14:02:34.385Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410312133Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50072024-02-04T14:02:34.385Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410364529Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50082024-02-04T14:02:34.385Z{"msg":"[0]R flush_numbers[0..12]: [70, 73, 73, 73, 58, 72, 72, 70, 70, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410390569Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50092024-02-04T14:02:34.385Z{"msg":"[0]R generation[0..12]: [43, 45, 45, 45, 35, 44, 44, 43, 43, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410414817Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50102024-02-04T14:02:34.385Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410439398Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50112024-02-04T14:02:34.385Z{"msg":"[1]R flush_numbers[0..12]: [70, 73, 73, 73, 58, 72, 72, 70, 70, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410462788Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50122024-02-04T14:02:34.385Z{"msg":"[1]R generation[0..12]: [43, 45, 45, 45, 35, 44, 44, 43, 43, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410504854Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50132024-02-04T14:02:34.385Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410530397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50142024-02-04T14:02:34.385Z{"msg":"[2]R flush_numbers[0..12]: [70, 73, 73, 73, 58, 72, 72, 70, 70, 67, 68, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410554075Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50152024-02-04T14:02:34.385Z{"msg":"[2]R generation[0..12]: [43, 45, 45, 45, 35, 44, 44, 43, 43, 40, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410604257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50162024-02-04T14:02:34.385Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
50172024-02-04T14:02:34.385Z,"time":"2024-02-04T14:02:33.410629273Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50182024-02-04T14:02:34.385Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410652947Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50192024-02-04T14:02:34.385Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410706723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50202024-02-04T14:02:34.385Z{"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.41072949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50212024-02-04T14:02:34.385Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410751586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50222024-02-04T14:02:34.385Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410800883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50232024-02-04T14:02:34.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.4108224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50242024-02-04T14:02:34.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410844968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50252024-02-04T14:02:34.385Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410867067Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50262024-02-04T14:02:34.385Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410908263Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50272024-02-04T14:02:34.385Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410930064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50282024-02-04T14:02:34.386Z{"msg":"9a08d979-545d-400b-964d-43bcd29ff99d is now active with session: f56a69d3-7a1d-46e6-a9f8-1447eea9812d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410951984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50292024-02-04T14:02:34.386Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.410991483Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313,"session_id":"f56a69d3-7a1d-46e6-a9f8-1447eea9812d"}
50302024-02-04T14:02:34.386Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:33.411013387Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313}
50312024-02-04T14:02:34.386ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50322024-02-04T14:02:34.386ZLoading write count information from file "/tmp/test_repair_verify.data"
50332024-02-04T14:02:34.386ZRun Repair workload
50342024-02-04T14:02:34.386Z01/30 Write block 349 len 4096 data: 3
50352024-02-04T14:02:34.389Z02/30 Write block 203 len 12288 data: 5 4 3
50362024-02-04T14:02:34.389Z03/30 Write block 10 len 8192 data: 2 2
50372024-02-04T14:02:34.390Z04/30 Read block 469 len 8192
50382024-02-04T14:02:34.390Z05/30 Flush
50392024-02-04T14:02:34.390Z06/30 Write block 185 len 4096 data: 7
50402024-02-04T14:02:34.390Z07/30 Write block 570 len 12288 data: 4 4 5
50412024-02-04T14:02:34.390Z08/30 Read block 280 len 16384
50422024-02-04T14:02:34.390Z09/30 Read block 7 len 20480
50432024-02-04T14:02:34.393Z10/30 Read block 380 len 24576
50442024-02-04T14:02:34.393Z11/30 Read block 507 len 28672
50452024-02-04T14:02:34.397Z12/30 Read block 344 len 12288
50462024-02-04T14:02:34.397Z13/30 Read block 590 len 20480
50472024-02-04T14:02:34.400Z14/30 Write block 441 len 32768 data: 3 2 2 2 2 2 2 3
50482024-02-04T14:02:34.400Z15/30 Write block 247 len 20480 data: 5 6 8 9 8
50492024-02-04T14:02:34.400Z16/30 Read block 330 len 12288
50502024-02-04T14:02:34.404Z17/30 Read block 372 len 40960
50512024-02-04T14:02:34.404Z18/30 Read block 175 len 8192
50522024-02-04T14:02:34.404Z19/30 Write block 4 len 24576 data: 4 4 4 4 3 4
50532024-02-04T14:02:34.407Z20/30 Write block 492 len 16384 data: 9 9 7 7
50542024-02-04T14:02:34.408Z21/30 Read block 157 len 32768
50552024-02-04T14:02:34.408Z22/30 Write block 553 len 8192 data: 4 4
50562024-02-04T14:02:34.411Z23/30 Write block 378 len 16384 data: 6 6 7 5
50572024-02-04T14:02:34.411Z24/30 Read block 354 len 8192
50582024-02-04T14:02:34.414Z25/30 Write block 393 len 40960 data: 6 6 4 4 5 4 4 5 4 4
50592024-02-04T14:02:34.414Z26/30 Write block 349 len 4096 data: 4
50602024-02-04T14:02:34.414Z27/30 Read block 440 len 8192
50612024-02-04T14:02:34.414Z28/30 Read block 521 len 36864
50622024-02-04T14:02:34.417Z29/30 Read block 430 len 28672
50632024-02-04T14:02:34.417Z30/30 Read block 479 len 32768
50642024-02-04T14:02:34.422Z----------------------------------------------------------------
50652024-02-04T14:02:34.423Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50662024-02-04T14:02:34.423ZStates: Active Active Active
50672024-02-04T14:02:34.423ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50682024-02-04T14:02:34.423Z 1 Acked 1000 Write 1 Done Done Done false
50692024-02-04T14:02:34.423Z 2 Acked 1001 Write 3 Done Done Done false
50702024-02-04T14:02:34.423Z 3 Acked 1002 Write 2 Done Done Done false
50712024-02-04T14:02:34.423Z 4 Acked 1003 Read 2 Done Sent Done false
50722024-02-04T14:02:34.423Z 5 Acked 1004 Flush 0 Done Sent Done false
50732024-02-04T14:02:34.423Z 6 Acked 1005 Write 1 Done Sent Done false
50742024-02-04T14:02:34.423Z 7 Acked 1006 Write 3 Done Sent Done false
50752024-02-04T14:02:34.423Z 8 Acked 1007 Read 4 Done Sent Done false
50762024-02-04T14:02:34.423Z 9 Acked 1008 Read 5 Done Sent Done false
50772024-02-04T14:02:34.423Z 10 Acked 1009 Read 6 Done Sent Done false
50782024-02-04T14:02:34.423Z 11 Acked 1010 Read 7 Done Sent Done false
50792024-02-04T14:02:34.423Z 12 Acked 1011 Read 3 Done Sent Done false
50802024-02-04T14:02:34.424Z 13 Acked 1012 Read 5 Done Sent Done false
50812024-02-04T14:02:34.424Z 14 Acked 1013 Write 8 Done Sent Done false
50822024-02-04T14:02:34.424Z 15 Acked 1014 Write 5 Done Sent Done false
50832024-02-04T14:02:34.424Z 16 Acked 1015 Read 3 Done Sent Done false
50842024-02-04T14:02:34.424Z 17 Acked 1016 Read 10 Done Sent Done false
50852024-02-04T14:02:34.424Z 18 Acked 1017 Read 2 Done Sent Done false
50862024-02-04T14:02:34.424Z 19 Acked 1018 Write 6 Done Sent Done false
50872024-02-04T14:02:34.424Z 20 Acked 1019 Write 4 Done Sent Done false
50882024-02-04T14:02:34.424Z 21 Acked 1020 Read 8 Done Sent Done false
50892024-02-04T14:02:34.424Z 22 Acked 1021 Write 2 Done Sent Done false
50902024-02-04T14:02:34.424Z 23 Acked 1022 Write 4 Done Sent Done false
50912024-02-04T14:02:34.424Z 24 Acked 1023 Read 2 Done Sent Done false
50922024-02-04T14:02:34.424Z 25 Acked 1024 Write 10 Done Sent Done false
50932024-02-04T14:02:34.424Z 26 Acked 1025 Write 1 Done Sent Done false
50942024-02-04T14:02:34.424Z 27 Acked 1026 Read 2 Done Sent Done false
50952024-02-04T14:02:34.425Z 28 Acked 1027 Read 9 Done Sent Done false
50962024-02-04T14:02:34.425Z 29 Acked 1028 Read 7 Done Sent Done false
50972024-02-04T14:02:34.425Z 30 Acked 1029 Read 8 Done Sent Done false
50982024-02-04T14:02:34.425Z STATES DS:0 DS:1 DS:2 TOTAL
50992024-02-04T14:02:34.425Z New 0 0 0 0
51002024-02-04T14:02:34.425Z Sent 0 27 0 27
51012024-02-04T14:02:34.425Z Done 30 3 30 63
51022024-02-04T14:02:34.425Z Skipped 0 0 0 0
51032024-02-04T14:02:34.425Z Error 0 0 0 0
51042024-02-04T14:02:34.425ZLast Flush: 1004 0 1004
51052024-02-04T14:02:34.425ZDownstairs last five completed:
51062024-02-04T14:02:34.425ZUpstairs last five completed: 30 29 28 27 26
51072024-02-04T14:02:34.425Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:33.452841079Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1313}
51082024-02-04T14:02:34.425ZWrote out file "/tmp/test_repair_verify.data"
51092024-02-04T14:02:34.429Z
51102024-02-04T14:02:34.432ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51112024-02-04T14:02:34.459ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51122024-02-04T14:02:34.459Z 0 000-019 46 43 46 74 70 74 T T T <---
51132024-02-04T14:02:34.459Z 1 020-039 45 45 45 73 73 73 F F F
51142024-02-04T14:02:34.459Z 2 040-059 45 45 45 73 73 73 F F F
51152024-02-04T14:02:34.459Z 3 060-079 45 45 45 73 73 73 F F F
51162024-02-04T14:02:34.463Z 4 080-099 35 35 35 58 58 58 F F F
51172024-02-04T14:02:34.463Z 5 100-119 44 44 44 72 72 72 F F F
51182024-02-04T14:02:34.463Z 6 120-139 44 44 44 72 72 72 F F F
51192024-02-04T14:02:34.463Z 7 140-159 43 43 43 70 70 70 F F F
51202024-02-04T14:02:34.463Z 8 160-179 43 43 43 70 70 70 F F F
51212024-02-04T14:02:34.463Z 9 180-199 40 40 40 67 67 67 T F T <---
51222024-02-04T14:02:34.463Z 10 200-219 46 41 46 74 68 74 F T F <---
51232024-02-04T14:02:34.463Z 11 220-239 40 40 40 67 67 67 F F F
51242024-02-04T14:02:34.463Z 12 240-259 44 44 44 72 72 72 T F T <---
51252024-02-04T14:02:34.463Z 13 260-279 44 44 44 72 72 72 F F F
51262024-02-04T14:02:34.463Z 14 280-299 45 45 45 73 73 73 F F F
51272024-02-04T14:02:34.463Z 15 300-319 45 45 45 73 73 73 F F F
51282024-02-04T14:02:34.463Z 16 320-339 40 40 40 67 67 67 F F F
51292024-02-04T14:02:34.463Z 17 340-359 46 38 46 74 62 74 T T T <---
51302024-02-04T14:02:34.463Z 18 360-379 43 43 43 70 70 70 T F T <---
51312024-02-04T14:02:34.463Z 19 380-399 44 44 44 71 71 71 T F T <---
51322024-02-04T14:02:34.463Z 20 400-419 43 43 43 70 70 70 T F T <---
51332024-02-04T14:02:34.463Z 21 420-439 43 43 43 70 70 70 F F F
51342024-02-04T14:02:34.463Z 22 440-459 35 35 35 58 58 58 T F T <---
51352024-02-04T14:02:34.463Z 23 460-479 44 44 44 72 72 72 F F F
51362024-02-04T14:02:34.463Z 24 480-499 44 44 44 72 72 72 T F T <---
51372024-02-04T14:02:34.463Z 25 500-519 44 44 44 71 71 71 F F F
51382024-02-04T14:02:34.464Z 26 520-539 44 44 44 71 71 71 F F F
51392024-02-04T14:02:34.464Z 27 540-559 43 43 43 70 70 70 T F T <---
51402024-02-04T14:02:34.464Z 28 560-579 44 44 44 71 71 71 T F T <---
51412024-02-04T14:02:34.464Z 29 580-599 42 42 42 69 69 69 F F F
51422024-02-04T14:02:34.464ZMax gen: 46, Max flush: 74
51432024-02-04T14:02:34.464ZError: Difference in extent metadata found!
51442024-02-04T14:02:34.464ZA Difference in extent metadata is expected here
51452024-02-04T14:02:34.464ZOn loop 23
51462024-02-04T14:02:34.464Z
51472024-02-04T14:02:34.467ZVerifying data now
51482024-02-04T14:02:39.541ZNo change after verify
51492024-02-04T14:02:39.541ZLoop: 23 Downstairs dump after verify (and repair):
51502024-02-04T14:02:39.564ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51512024-02-04T14:02:39.564Z 0 000-019 47 47 47 75 75 75 F F F
51522024-02-04T14:02:39.564Z 1 020-039 45 45 45 73 73 73 F F F
51532024-02-04T14:02:39.564Z 2 040-059 45 45 45 73 73 73 F F F
51542024-02-04T14:02:39.564Z 3 060-079 45 45 45 73 73 73 F F F
51552024-02-04T14:02:39.564Z 4 080-099 35 35 35 58 58 58 F F F
51562024-02-04T14:02:39.564Z 5 100-119 44 44 44 72 72 72 F F F
51572024-02-04T14:02:39.564Z 6 120-139 44 44 44 72 72 72 F F F
51582024-02-04T14:02:39.564Z 7 140-159 43 43 43 70 70 70 F F F
51592024-02-04T14:02:39.564Z 8 160-179 43 43 43 70 70 70 F F F
51602024-02-04T14:02:39.564Z 9 180-199 47 47 47 75 75 75 F F F
51612024-02-04T14:02:39.564Z 10 200-219 46 46 46 74 74 74 F F F
51622024-02-04T14:02:39.564Z 11 220-239 40 40 40 67 67 67 F F F
51632024-02-04T14:02:39.564Z 12 240-259 47 47 47 75 75 75 F F F
51642024-02-04T14:02:39.564Z 13 260-279 44 44 44 72 72 72 F F F
51652024-02-04T14:02:39.564Z 14 280-299 45 45 45 73 73 73 F F F
51662024-02-04T14:02:39.564Z 15 300-319 45 45 45 73 73 73 F F F
51672024-02-04T14:02:39.568Z 16 320-339 40 40 40 67 67 67 F F F
51682024-02-04T14:02:39.568Z 17 340-359 47 47 47 75 75 75 F F F
51692024-02-04T14:02:39.568Z 18 360-379 47 47 47 75 75 75 F F F
51702024-02-04T14:02:39.568Z 19 380-399 47 47 47 75 75 75 F F F
51712024-02-04T14:02:39.568Z 20 400-419 47 47 47 75 75 75 F F F
51722024-02-04T14:02:39.568Z 21 420-439 43 43 43 70 70 70 F F F
51732024-02-04T14:02:39.568Z 22 440-459 47 47 47 75 75 75 F F F
51742024-02-04T14:02:39.568Z 23 460-479 44 44 44 72 72 72 F F F
51752024-02-04T14:02:39.568Z 24 480-499 47 47 47 75 75 75 F F F
51762024-02-04T14:02:39.568Z 25 500-519 44 44 44 71 71 71 F F F
51772024-02-04T14:02:39.568Z 26 520-539 44 44 44 71 71 71 F F F
51782024-02-04T14:02:39.568Z 27 540-559 47 47 47 75 75 75 F F F
51792024-02-04T14:02:39.568Z 28 560-579 47 47 47 75 75 75 F F F
51802024-02-04T14:02:39.568Z 29 580-599 42 42 42 69 69 69 F F F
51812024-02-04T14:02:39.568ZMax gen: 47, Max flush: 75
51822024-02-04T14:02:39.571Z
51832024-02-04T14:02:39.572ZBegin loop 24
51842024-02-04T14:02:39.572ZDownstairs to restart: 1
51852024-02-04T14:02:39.599Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.632106045Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321}
51862024-02-04T14:02:39.599Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.632579842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321}
51872024-02-04T14:02:39.599Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.632613842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321}
51882024-02-04T14:02:39.599Z{"msg":"Crucible a69144bd-8f6a-4980-af31-6f04e97c4c34 has session id: 3b9ddb39-79bf-4d65-88e5-bce7b6f59990","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.632685841Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51892024-02-04T14:02:39.602Z{"msg":"Upstairs opts: Upstairs UUID: a69144bd-8f6a-4980-af31-6f04e97c4c34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.632737582Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51902024-02-04T14:02:39.603Z{"msg":"Crucible stats registered with UUID: a69144bd-8f6a-4980-af31-6f04e97c4c34","v":0,"name":"crucible","level":30Crucible runtime is spawned
51912024-02-04T14:02:39.603Z,"time":"2024-02-04T14:02:38.632906292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51922024-02-04T14:02:39.603Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.633275953Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321}
51932024-02-04T14:02:39.603Z{"msg":"a69144bd-8f6a-4980-af31-6f04e97c4c34 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.633470837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51942024-02-04T14:02:39.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.633596465Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51952024-02-04T14:02:39.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.633642316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51962024-02-04T14:02:39.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.633711797Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51972024-02-04T14:02:39.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.633969639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51982024-02-04T14:02:39.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.634176729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
51992024-02-04T14:02:39.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.634288566Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52002024-02-04T14:02:39.603Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.63433395Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52012024-02-04T14:02:39.603Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.634402758Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52022024-02-04T14:02:39.603Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.634431378Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52032024-02-04T14:02:39.603Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.634474152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52042024-02-04T14:02:39.603Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.634499128Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52052024-02-04T14:02:39.603Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.634550079Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52062024-02-04T14:02:39.603Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.635573642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52072024-02-04T14:02:39.603Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.635690664Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52082024-02-04T14:02:39.603Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.635879798Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52092024-02-04T14:02:39.603Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.635973059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52102024-02-04T14:02:39.607Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.636580577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52112024-02-04T14:02:39.608Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.636796763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52122024-02-04T14:02:39.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637007825Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52132024-02-04T14:02:39.608Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637061667Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52142024-02-04T14:02:39.608Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637423535Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52152024-02-04T14:02:39.608Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637481027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52162024-02-04T14:02:39.608Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637509995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52172024-02-04T14:02:39.608Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637531148Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52182024-02-04T14:02:39.608Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637578958Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52192024-02-04T14:02:39.608Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637605499Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52202024-02-04T14:02:39.608Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.637951173Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52212024-02-04T14:02:39.608Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.63865833Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52222024-02-04T14:02:39.608Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638718106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52232024-02-04T14:02:39.608Z{"msg":"[0]R flush_numbers[0..12]: [75, 73, 73, 73, 58, 72, 72, 70, 70, 75, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638744857Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52242024-02-04T14:02:39.608Z{"msg":"[0]R generation[0..12]: [47, 45, 45, 45, 35, 44, 44, 43, 43, 47, 46, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638768334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52252024-02-04T14:02:39.608Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638811648Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52262024-02-04T14:02:39.608Z{"msg":"[1]R flush_numbers[0..12]: [75, 73, 73, 73, 58, 72, 72, 70, 70, 75, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638836954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52272024-02-04T14:02:39.608Z{"msg":"[1]R generation[0..12]: [47, 45, 45, 45, 35, 44, 44, 43, 43, 47, 46, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638860306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52282024-02-04T14:02:39.608Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638902159Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52292024-02-04T14:02:39.608Z{"msg":"[2]R flush_numbers[0..12]: [75, 73, 73, 73, 58, 72, 72, 70, 70, 75, 74, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638927389Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52302024-02-04T14:02:39.608Z{"msg":"[2]R generation[0..12]: [47, 45, 45, 45, 35, 44, 44, 43, 43, 47, 46, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638950864Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52312024-02-04T14:02:39.608Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.638993164Z","hostname":"Wait for a query_work_queue command to finish before sending IO
52322024-02-04T14:02:39.608Zip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52332024-02-04T14:02:39.609Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639019979Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52342024-02-04T14:02:39.609Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639044028Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52352024-02-04T14:02:39.609Z{"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639065706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52362024-02-04T14:02:39.609Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.63908755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52372024-02-04T14:02:39.609Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639109991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52382024-02-04T14:02:39.609Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639130058Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52392024-02-04T14:02:39.609Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639182148Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52402024-02-04T14:02:39.609Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.63920586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52412024-02-04T14:02:39.609Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639228027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52422024-02-04T14:02:39.609Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639268044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52432024-02-04T14:02:39.609Z{"msg":"a69144bd-8f6a-4980-af31-6f04e97c4c34 is now active with session: 3b9ddb39-79bf-4d65-88e5-bce7b6f59990","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639291026Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52442024-02-04T14:02:39.609Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639311429Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321,"session_id":"3b9ddb39-79bf-4d65-88e5-bce7b6f59990"}
52452024-02-04T14:02:39.609Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:38.639331812Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321}
52462024-02-04T14:02:39.609ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52472024-02-04T14:02:39.609ZLoading write count information from file "/tmp/test_repair_verify.data"
52482024-02-04T14:02:39.609ZRun Repair workload
52492024-02-04T14:02:39.616Z01/30 Write block 421 len 4096 data: 7
52502024-02-04T14:02:39.616Z02/30 Read block 546 len 32768
52512024-02-04T14:02:39.616Z03/30 Read block 71 len 20480
52522024-02-04T14:02:39.616Z04/30 Read block 149 len 12288
52532024-02-04T14:02:39.619Z05/30 Write block 475 len 20480 data: 7 5 5 5 5
52542024-02-04T14:02:39.619Z06/30 Read block 535 len 24576
52552024-02-04T14:02:39.619Z07/30 Read block 331 len 16384
52562024-02-04T14:02:39.619Z08/30 Read block 187 len 28672
52572024-02-04T14:02:39.622Z09/30 Read block 143 len 36864
52582024-02-04T14:02:39.622Z10/30 Write block 48 len 24576 data: 4 3 2 3 3 3
52592024-02-04T14:02:39.625Z11/30 Read block 70 len 40960
52602024-02-04T14:02:39.625Z12/30 Read block 500 len 36864
52612024-02-04T14:02:39.628Z13/30 Write block 506 len 20480 data: 6 6 6 7 6
52622024-02-04T14:02:39.628Z14/30 Read block 214 len 32768
52632024-02-04T14:02:39.631Z15/30 Read block 297 len 4096
52642024-02-04T14:02:39.631Z16/30 Write block 197 len 24576 data: 4 6 6 6 6 5
52652024-02-04T14:02:39.634Z17/30 Read block 325 len 16384
52662024-02-04T14:02:39.634Z18/30 Read block 351 len 20480
52672024-02-04T14:02:39.637Z19/30 Read block 29 len 4096
52682024-02-04T14:02:39.637Z20/30 Write block 420 len 4096 data: 7
52692024-02-04T14:02:39.637Z21/30 Write block 560 len 32768 data: 6 7 6 6 7 6 6 5
52702024-02-04T14:02:39.640Z22/30 Read block 347 len 28672
52712024-02-04T14:02:39.640Z23/30 Read block 239 len 40960
52722024-02-04T14:02:39.646Z24/30 Read block 292 len 12288
52732024-02-04T14:02:39.648Z25/30 Write block 144 len 36864 data: 4 4 4 3 3 3 3 3 3
52742024-02-04T14:02:39.649Z26/30 Write block 202 len 20480 data: 6 6 5 4 4
52752024-02-04T14:02:39.649Z27/30 Write block 383 len 28672 data: 4 4 3 3 4 5 5
52762024-02-04T14:02:39.649Z28/30 Read block 340 len 32768
52772024-02-04T14:02:39.649Z29/30 Read block 367 len 36864
52782024-02-04T14:02:39.651Z30/30 Read block 304 len 28672
52792024-02-04T14:02:39.687Z----------------------------------------------------------------
52802024-02-04T14:02:39.687Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52812024-02-04T14:02:39.687ZStates: Active Active Active
52822024-02-04T14:02:39.687ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52832024-02-04T14:02:39.687Z 1 Acked 1000 Write 1 Done Done Done false
52842024-02-04T14:02:39.687Z 2 Acked 1001 Read 8 Done Done Done false
52852024-02-04T14:02:39.687Z 3 Acked 1002 Read 5 Done Done Done false
52862024-02-04T14:02:39.687Z 4 Acked 1003 Read 3 Done Sent Done false
52872024-02-04T14:02:39.687Z 5 Acked 1004 Write 5 Done Sent Done false
52882024-02-04T14:02:39.687Z 6 Acked 1005 Read 6 Done Sent Done false
52892024-02-04T14:02:39.687Z 7 Acked 1006 Read 4 Done Sent Done false
52902024-02-04T14:02:39.688Z 8 Acked 1007 Read 7 Done Sent Done false
52912024-02-04T14:02:39.688Z 9 Acked 1008 Read 9 Done Sent Done false
52922024-02-04T14:02:39.688Z 10 Acked 1009 Write 6 Done Sent Done false
52932024-02-04T14:02:39.688Z 11 Acked 1010 Read 10 Done Sent Done false
52942024-02-04T14:02:39.688Z 12 Acked 1011 Read 9 Done Sent Done false
52952024-02-04T14:02:39.688Z 13 Acked 1012 Write 5 Done Sent Done false
52962024-02-04T14:02:39.688Z 14 Acked 1013 Read 8 Done Sent Done false
52972024-02-04T14:02:39.688Z 15 Acked 1014 Read 1 Done Sent Done false
52982024-02-04T14:02:39.688Z 16 Acked 1015 Write 6 Done Sent Done false
52992024-02-04T14:02:39.688Z 17 Acked 1016 Read 4 Done Sent Done false
53002024-02-04T14:02:39.688Z 18 Acked 1017 Read 5 Done Sent Done false
53012024-02-04T14:02:39.688Z 19 Acked 1018 Read 1 Done Sent Done false
53022024-02-04T14:02:39.688Z 20 Acked 1019 Write 1 Done Sent Done false
53032024-02-04T14:02:39.688Z 21 Acked 1020 Write 8 Done Sent Done false
53042024-02-04T14:02:39.688Z 22 Acked 1021 Read 7 Done Sent Done false
53052024-02-04T14:02:39.688Z 23 Acked 1022 Read 10 Done Sent Done false
53062024-02-04T14:02:39.688Z 24 Acked 1023 Read 3 Done Sent Done false
53072024-02-04T14:02:39.688Z 25 Acked 1024 Write 9 Done Sent Done false
53082024-02-04T14:02:39.688Z 26 Acked 1025 Write 5 Done Sent Done false
53092024-02-04T14:02:39.688Z 27 Acked 1026 Write 7 Done Sent Done false
53102024-02-04T14:02:39.688Z 28 Acked 1027 Read 8 Done Sent Done false
53112024-02-04T14:02:39.688Z 29 Acked 1028 Read 9 Done Sent Done false
53122024-02-04T14:02:39.688Z 30 Acked 1029 Read 7 Done Sent Done false
53132024-02-04T14:02:39.688Z STATES DS:0 DS:1 DS:2 TOTAL
53142024-02-04T14:02:39.688Z New 0 0 0 0
53152024-02-04T14:02:39.689Z Sent 0 27 0 27
53162024-02-04T14:02:39.689Z Done 30 3 30 63
53172024-02-04T14:02:39.689Z Skipped 0 0 0 0
53182024-02-04T14:02:39.689Z Error 0 0 0 0
53192024-02-04T14:02:39.689ZLast Flush: 0 0 0
53202024-02-04T14:02:39.689ZDownstairs last five completed:
53212024-02-04T14:02:39.689ZUpstairs last five completed: 30 29 28 27 26
53222024-02-04T14:02:39.689Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:38.686282402Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1321}
53232024-02-04T14:02:39.689ZWrote out file "/tmp/test_repair_verify.data"
53242024-02-04T14:02:39.689Z
53252024-02-04T14:02:39.689ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53262024-02-04T14:02:39.697ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53272024-02-04T14:02:39.697Z 0 000-019 47 47 47 75 75 75 F F F
53282024-02-04T14:02:39.697Z 1 020-039 45 45 45 73 73 73 F F F
53292024-02-04T14:02:39.697Z 2 040-059 45 45 45 73 73 73 T F T <---
53302024-02-04T14:02:39.697Z 3 060-079 45 45 45 73 73 73 F F F
53312024-02-04T14:02:39.697Z 4 080-099 35 35 35 58 58 58 F F F
53322024-02-04T14:02:39.697Z 5 100-119 44 44 44 72 72 72 F F F
53332024-02-04T14:02:39.697Z 6 120-139 44 44 44 72 72 72 F F F
53342024-02-04T14:02:39.697Z 7 140-159 43 43 43 70 70 70 T F T <---
53352024-02-04T14:02:39.697Z 8 160-179 43 43 43 70 70 70 F F F
53362024-02-04T14:02:39.697Z 9 180-199 47 47 47 75 75 75 T F T <---
53372024-02-04T14:02:39.697Z 10 200-219 46 46 46 74 74 74 T F T <---
53382024-02-04T14:02:39.700Z 11 220-239 40 40 40 67 67 67 F F F
53392024-02-04T14:02:39.700Z 12 240-259 47 47 47 75 75 75 F F F
53402024-02-04T14:02:39.700Z 13 260-279 44 44 44 72 72 72 F F F
53412024-02-04T14:02:39.701Z 14 280-299 45 45 45 73 73 73 F F F
53422024-02-04T14:02:39.701Z 15 300-319 45 45 45 73 73 73 F F F
53432024-02-04T14:02:39.701Z 16 320-339 40 40 40 67 67 67 F F F
53442024-02-04T14:02:39.701Z 17 340-359 47 47 47 75 75 75 F F F
53452024-02-04T14:02:39.701Z 18 360-379 47 47 47 75 75 75 F F F
53462024-02-04T14:02:39.701Z 19 380-399 47 47 47 75 75 75 T F T <---
53472024-02-04T14:02:39.701Z 20 400-419 47 47 47 75 75 75 F F F
53482024-02-04T14:02:39.701Z 21 420-439 43 43 43 70 70 70 T T T
53492024-02-04T14:02:39.701Z 22 440-459 47 47 47 75 75 75 F F F
53502024-02-04T14:02:39.701Z 23 460-479 44 44 44 72 72 72 T F T <---
53512024-02-04T14:02:39.701Z 24 480-499 47 47 47 75 75 75 F F F
53522024-02-04T14:02:39.701Z 25 500-519 44 44 44 71 71 71 T F T <---
53532024-02-04T14:02:39.701Z 26 520-539 44 44 44 71 71 71 F F F
53542024-02-04T14:02:39.701Z 27 540-559 47 47 47 75 75 75 F F F
53552024-02-04T14:02:39.701Z 28 560-579 47 47 47 75 75 75 T F T <---
53562024-02-04T14:02:39.701Z 29 580-599 42 42 42 69 69 69 F F F
53572024-02-04T14:02:39.701ZMax gen: 47, Max flush: 75
53582024-02-04T14:02:39.701ZError: Difference in extent metadata found!
53592024-02-04T14:02:39.704ZA Difference in extent metadata is expected here
53602024-02-04T14:02:39.704ZOn loop 24
53612024-02-04T14:02:39.704Z
53622024-02-04T14:02:39.708ZVerifying data now
53632024-02-04T14:02:44.577ZNo change after verify
53642024-02-04T14:02:44.577ZLoop: 24 Downstairs dump after verify (and repair):
53652024-02-04T14:02:44.601ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53662024-02-04T14:02:44.601Z 0 000-019 47 47 47 75 75 75 F F F
53672024-02-04T14:02:44.601Z 1 020-039 45 45 45 73 73 73 F F F
53682024-02-04T14:02:44.601Z 2 040-059 48 48 48 76 76 76 F F F
53692024-02-04T14:02:44.601Z 3 060-079 45 45 45 73 73 73 F F F
53702024-02-04T14:02:44.601Z 4 080-099 35 35 35 58 58 58 F F F
53712024-02-04T14:02:44.601Z 5 100-119 44 44 44 72 72 72 F F F
53722024-02-04T14:02:44.601Z 6 120-139 44 44 44 72 72 72 F F F
53732024-02-04T14:02:44.601Z 7 140-159 48 48 48 76 76 76 F F F
53742024-02-04T14:02:44.601Z 8 160-179 43 43 43 70 70 70 F F F
53752024-02-04T14:02:44.601Z 9 180-199 48 48 48 76 76 76 F F F
53762024-02-04T14:02:44.601Z 10 200-219 48 48 48 76 76 76 F F F
53772024-02-04T14:02:44.602Z 11 220-239 40 40 40 67 67 67 F F F
53782024-02-04T14:02:44.602Z 12 240-259 47 47 47 75 75 75 F F F
53792024-02-04T14:02:44.602Z 13 260-279 44 44 44 72 72 72 F F F
53802024-02-04T14:02:44.602Z 14 280-299 45 45 45 73 73 73 F F F
53812024-02-04T14:02:44.602Z 15 300-319 45 45 45 73 73 73 F F F
53822024-02-04T14:02:44.602Z 16 320-339 40 40 40 67 67 67 F F F
53832024-02-04T14:02:44.602Z 17 340-359 47 47 47 75 75 75 F F F
53842024-02-04T14:02:44.602Z 18 360-379 47 47 47 75 75 75 F F F
53852024-02-04T14:02:44.605Z 19 380-399 48 48 48 76 76 76 F F F
53862024-02-04T14:02:44.605Z 20 400-419 47 47 47 75 75 75 F F F
53872024-02-04T14:02:44.605Z 21 420-439 48 48 48 76 76 76 F F F
53882024-02-04T14:02:44.605Z 22 440-459 47 47 47 75 75 75 F F F
53892024-02-04T14:02:44.605Z 23 460-479 48 48 48 76 76 76 F F F
53902024-02-04T14:02:44.605Z 24 480-499 47 47 47 75 75 75 F F F
53912024-02-04T14:02:44.605Z 25 500-519 48 48 48 76 76 76 F F F
53922024-02-04T14:02:44.605Z 26 520-539 44 44 44 71 71 71 F F F
53932024-02-04T14:02:44.605Z 27 540-559 47 47 47 75 75 75 F F F
53942024-02-04T14:02:44.605Z 28 560-579 48 48 48 76 76 76 F F F
53952024-02-04T14:02:44.605Z 29 580-599 42 42 42 69 69 69 F F F
53962024-02-04T14:02:44.605ZMax gen: 48, Max flush: 76
53972024-02-04T14:02:44.609Z
53982024-02-04T14:02:44.609ZBegin loop 25
53992024-02-04T14:02:44.609ZDownstairs to restart: 0
54002024-02-04T14:02:44.641Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.674536868Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329}
54012024-02-04T14:02:44.641Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675007735Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329}
54022024-02-04T14:02:44.642Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675037523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329}
54032024-02-04T14:02:44.642Z{"msg":"Crucible 92f2a049-36ae-4d9f-88a4-efccfe1ca918 has session id: 43d6c820-f01e-4e66-ae03-3a090bbf2a53","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675063396Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54042024-02-04T14:02:44.642Z{"msg":"Upstairs opts: Upstairs UUID: 92f2a049-36ae-4d9f-88a4-efccfe1ca918, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675115091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54052024-02-04T14:02:44.642Z{"msg":"Crucible stats registered with UUID: 92f2a049-36ae-4d9f-88a4-efccfe1ca918","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675161064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54062024-02-04T14:02:44.645ZCrucible runtime is spawned
54072024-02-04T14:02:44.645Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675482799Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329}
54082024-02-04T14:02:44.645Z{"msg":"92f2a049-36ae-4d9f-88a4-efccfe1ca918 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675616968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54092024-02-04T14:02:44.645Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675686605Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54102024-02-04T14:02:44.645Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675762829Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54112024-02-04T14:02:44.646Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675804971Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54122024-02-04T14:02:44.646Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675863851Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54132024-02-04T14:02:44.646Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675895859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54142024-02-04T14:02:44.646Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.675996404Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54152024-02-04T14:02:44.646Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.676299197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"io task","client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54162024-02-04T14:02:44.646Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.67637331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"io task","client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54172024-02-04T14:02:44.646Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.676422855Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"io task","client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54182024-02-04T14:02:44.646Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.676493479Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"io task","client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54192024-02-04T14:02:44.646Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.676531448Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"io task","client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54202024-02-04T14:02:44.646Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.676600193Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"io task","client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54212024-02-04T14:02:44.646Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.677502267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54222024-02-04T14:02:44.646Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.677596189Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54232024-02-04T14:02:44.646Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.67762763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54242024-02-04T14:02:44.646Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.677692481Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54252024-02-04T14:02:44.646Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.67863669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54262024-02-04T14:02:44.650Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.678983538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54272024-02-04T14:02:44.650Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.679504789Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54282024-02-04T14:02:44.650Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.679592238Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54292024-02-04T14:02:44.650Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.679639376Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54302024-02-04T14:02:44.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.67969407Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54312024-02-04T14:02:44.650Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.679717385Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54322024-02-04T14:02:44.650Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.679859669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54332024-02-04T14:02:44.650Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.679912645Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54342024-02-04T14:02:44.650Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.679937433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54352024-02-04T14:02:44.650Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.680447648Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54362024-02-04T14:02:44.650Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681103028Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54372024-02-04T14:02:44.650Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.68115681Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54382024-02-04T14:02:44.650Z{"msg":"[0]R flush_numbers[0..12]: [75, 73, 76, 73, 58, 72, 72, 76, 70, 76, 76, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681183109Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54392024-02-04T14:02:44.650Z{"msg":"[0]R generation[0..12]: [47, 45, 48, 45, 35, 44, 44, 48, 43, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681206915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54402024-02-04T14:02:44.651Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681250329Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54412024-02-04T14:02:44.651Z{"msg":"[1]R flush_numbers[0..12]: [75, 73, 76, 73, 58, 72, 72, 76, 70, 76, 76, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681275827Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54422024-02-04T14:02:44.651Z{"msg":"[1]R generation[0..12]: [47, 45, 48, 45, 35, 44, 44, 48, 43, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681318039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54432024-02-04T14:02:44.651Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681343425Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54442024-02-04T14:02:44.651Z{"Wait for a query_work_queue command to finish before sending IO
54452024-02-04T14:02:44.651Zmsg":"[2]R flush_numbers[0..12]: [75, 73, 76, 73, 58, 72, 72, 76, 70, 76, 76, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681389793Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54462024-02-04T14:02:44.651Z{"msg":"[2]R generation[0..12]: [47, 45, 48, 45, 35, 44, 44, 48, 43, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681414172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54472024-02-04T14:02:44.651Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681438063Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54482024-02-04T14:02:44.651Z{"msg":"Max found gen is 49","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681460197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54492024-02-04T14:02:44.651Z{"msg":"Generation requested: 50 >= found:49","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681482652Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54502024-02-04T14:02:44.651Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681533117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54512024-02-04T14:02:44.651Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681560079Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54522024-02-04T14:02:44.651Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681583382Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54532024-02-04T14:02:44.651Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681605506Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54542024-02-04T14:02:44.651Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681628056Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54552024-02-04T14:02:44.651Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681650275Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54562024-02-04T14:02:44.651Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681694513Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54572024-02-04T14:02:44.651Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681716647Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54582024-02-04T14:02:44.651Z{"msg":"92f2a049-36ae-4d9f-88a4-efccfe1ca918 is now active with session: 43d6c820-f01e-4e66-ae03-3a090bbf2a53","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681738737Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54592024-02-04T14:02:44.651Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681758779Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329,"session_id":"43d6c820-f01e-4e66-ae03-3a090bbf2a53"}
54602024-02-04T14:02:44.651Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:43.681779484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329}
54612024-02-04T14:02:44.651ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54622024-02-04T14:02:44.651ZLoading write count information from file "/tmp/test_repair_verify.data"
54632024-02-04T14:02:44.652ZRun Repair workload
54642024-02-04T14:02:44.655Z01/30 Write block 356 len 40960 data: 6 4 4 4 4 5 5 6 6 6
54652024-02-04T14:02:44.655Z02/30 Write block 373 len 36864 data: 3 6 7 6 5 7 7 8 6
54662024-02-04T14:02:44.655Z03/30 Write block 428 len 32768 data: 4 4 5 4 4 4 4 4
54672024-02-04T14:02:44.658Z04/30 Write block 110 len 12288 data: 6 7 9
54682024-02-04T14:02:44.658Z05/30 Read block 162 len 36864
54692024-02-04T14:02:44.660Z06/30 Read block 530 len 40960
54702024-02-04T14:02:44.663Z07/30 Read block 412 len 4096
54712024-02-04T14:02:44.663Z08/30 Write block 465 len 12288 data: 6 8 7
54722024-02-04T14:02:44.663Z09/30 Read block 63 len 12288
54732024-02-04T14:02:44.666Z10/30 Write block 255 len 20480 data: 6 5 5 5 5
54742024-02-04T14:02:44.666Z11/30 Write block 526 len 36864 data: 3 2 3 3 3 2 3 4 6
54752024-02-04T14:02:44.669Z12/30 Write block 557 len 4096 data: 6
54762024-02-04T14:02:44.669Z13/30 Read block 455 len 28672
54772024-02-04T14:02:44.672Z14/30 Write block 397 len 20480 data: 6 5 5 6 5
54782024-02-04T14:02:44.672Z15/30 Read block 571 len 40960
54792024-02-04T14:02:44.677Z16/30 Read block 322 len 32768
54802024-02-04T14:02:44.677Z17/30 Flush
54812024-02-04T14:02:44.680Z18/30 Write block 106 len 28672 data: 4 5 8 7 7 8 10
54822024-02-04T14:02:44.683Z19/30 Read block 58 len 40960
54832024-02-04T14:02:44.686Z20/30 Read block 267 len 40960
54842024-02-04T14:02:44.686Z21/30 Write block 307 len 4096 data: 4
54852024-02-04T14:02:44.689Z22/30 Write block 8 len 36864 data: 4 5 3 3 2 2 3 3 3
54862024-02-04T14:02:44.689Z23/30 Write block 114 len 40960 data: 8 8 8 8 5 5 5 5 5 5
54872024-02-04T14:02:44.691Z24/30 Read block 28 len 4096
54882024-02-04T14:02:44.691Z25/30 Write block 165 len 24576 data: 5 5 5 6 6 7
54892024-02-04T14:02:44.694Z26/30 Write block 575 len 28672 data: 5 6 6 5 8 9 9
54902024-02-04T14:02:44.694Z27/30 Read block 561 len 40960
54912024-02-04T14:02:44.697Z28/30 Write block 339 len 8192 data: 4 4
54922024-02-04T14:02:44.697Z29/30 Write block 33 len 16384 data: 4 4 5 7
54932024-02-04T14:02:44.700Z30/30 Read block 65 len 24576
54942024-02-04T14:02:44.701Z----------------------------------------------------------------
54952024-02-04T14:02:44.701Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
54962024-02-04T14:02:44.701ZStates: Active Active Active
54972024-02-04T14:02:44.701ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54982024-02-04T14:02:44.701Z 1 Acked 1000 Write 10 Sent Done Done false
54992024-02-04T14:02:44.701Z 2 Acked 1001 Write 9 Sent Done Done false
55002024-02-04T14:02:44.701Z 3 Acked 1002 Write 8 Sent Done Done false
55012024-02-04T14:02:44.701Z 4 Acked 1003 Write 3 Sent Done Done false
55022024-02-04T14:02:44.701Z 5 Acked 1004 Read 9 Sent Done Done false
55032024-02-04T14:02:44.701Z 6 Acked 1005 Read 10 Sent Done Done false
55042024-02-04T14:02:44.701Z 7 Acked 1006 Read 1 Sent Done Done false
55052024-02-04T14:02:44.701Z 8 Acked 1007 Write 3 Sent Done Done false
55062024-02-04T14:02:44.701Z 9 Acked 1008 Read 3 Sent Done Done false
55072024-02-04T14:02:44.701Z 10 Acked 1009 Write 5 Sent Done Done false
55082024-02-04T14:02:44.701Z 11 Acked 1010 Write 9 Sent Done Done false
55092024-02-04T14:02:44.701Z 12 Acked 1011 Write 1 Sent Done Done false
55102024-02-04T14:02:44.701Z 13 Acked 1012 Read 7 Sent Done Done false
55112024-02-04T14:02:44.701Z 14 Acked 1013 Write 5 Sent Done Done false
55122024-02-04T14:02:44.701Z 15 Acked 1014 Read 10 Sent Done Done false
55132024-02-04T14:02:44.701Z 16 Acked 1015 Read 8 Sent Done Done false
55142024-02-04T14:02:44.701Z 17 Acked 1016 Flush 0 Sent Done Done false
55152024-02-04T14:02:44.701Z 18 Acked 1017 Write 7 Sent Done Done false
55162024-02-04T14:02:44.701Z 19 Acked 1018 Read 10 Sent Done Done false
55172024-02-04T14:02:44.702Z 20 Acked 1019 Read 10 Sent Done Done false
55182024-02-04T14:02:44.702Z 21 Acked 1020 Write 1 Sent Done Done false
55192024-02-04T14:02:44.702Z 22 Acked 1021 Write 9 Sent Done Done false
55202024-02-04T14:02:44.702Z 23 Acked 1022 Write 10 Sent Done Done false
55212024-02-04T14:02:44.702Z 24 Acked 1023 Read 1 Sent Done Done false
55222024-02-04T14:02:44.702Z 25 Acked 1024 Write 6 Sent Done Done false
55232024-02-04T14:02:44.702Z 26 Acked 1025 Write 7 Sent Done Done false
55242024-02-04T14:02:44.702Z 27 Acked 1026 Read 10 Sent Done Done false
55252024-02-04T14:02:44.702Z 28 Acked 1027 Write 2 Sent Done Done false
55262024-02-04T14:02:44.702Z 29 Acked 1028 Write 4 Sent Done Done false
55272024-02-04T14:02:44.702Z 30 Acked 1029 Read 6 Sent Done Done false
55282024-02-04T14:02:44.702Z STATES DS:0 DS:1 DS:2 TOTAL
55292024-02-04T14:02:44.702Z New 0 0 0 0
55302024-02-04T14:02:44.702Z Sent 30 0 0 30
55312024-02-04T14:02:44.702Z Done 0 30 30 60
55322024-02-04T14:02:44.702Z Skipped 0 0 0 0
55332024-02-04T14:02:44.702Z Error 0 0 0 0
55342024-02-04T14:02:44.702ZLast Flush: 0 1016 1016
55352024-02-04T14:02:44.702ZDownstairs last five completed:
55362024-02-04T14:02:44.702ZUpstairs last five completed: 30 29 28 27 26
55372024-02-04T14:02:44.702Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:43.733619868Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1329}
55382024-02-04T14:02:44.702ZWrote out file "/tmp/test_repair_verify.data"
55392024-02-04T14:02:44.707Z
55402024-02-04T14:02:44.714ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55412024-02-04T14:02:44.743ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55422024-02-04T14:02:44.743Z 0 000-019 47 47 47 75 75 75 F T T <---
55432024-02-04T14:02:44.743Z 1 020-039 45 45 45 73 73 73 F T T <---
55442024-02-04T14:02:44.743Z 2 040-059 48 48 48 76 76 76 F F F
55452024-02-04T14:02:44.743Z 3 060-079 45 45 45 73 73 73 F F F
55462024-02-04T14:02:44.743Z 4 080-099 35 35 35 58 58 58 F F F
55472024-02-04T14:02:44.743Z 5 100-119 44 50 50 72 77 77 F T T <---
55482024-02-04T14:02:44.743Z 6 120-139 44 44 44 72 72 72 F T T <---
55492024-02-04T14:02:44.743Z 7 140-159 48 48 48 76 76 76 F F F
55502024-02-04T14:02:44.743Z 8 160-179 43 43 43 70 70 70 F T T <---
55512024-02-04T14:02:44.747Z 9 180-199 48 48 48 76 76 76 F F F
55522024-02-04T14:02:44.747Z 10 200-219 48 48 48 76 76 76 F F F
55532024-02-04T14:02:44.747Z 11 220-239 40 40 40 67 67 67 F F F
55542024-02-04T14:02:44.747Z 12 240-259 47 50 50 75 77 77 F F F <---
55552024-02-04T14:02:44.747Z 13 260-279 44 44 44 72 72 72 F F F
55562024-02-04T14:02:44.747Z 14 280-299 45 45 45 73 73 73 F F F
55572024-02-04T14:02:44.747Z 15 300-319 45 45 45 73 73 73 F T T <---
55582024-02-04T14:02:44.747Z 16 320-339 40 40 40 67 67 67 F T T <---
55592024-02-04T14:02:44.747Z 17 340-359 47 50 50 75 77 77 F T T <---
55602024-02-04T14:02:44.747Z 18 360-379 47 50 50 75 77 77 F F F <---
55612024-02-04T14:02:44.747Z 19 380-399 48 50 50 76 77 77 F F F <---
55622024-02-04T14:02:44.747Z 20 400-419 47 50 50 75 77 77 F F F <---
55632024-02-04T14:02:44.747Z 21 420-439 48 50 50 76 77 77 F F F <---
55642024-02-04T14:02:44.747Z 22 440-459 47 47 47 75 75 75 F F F
55652024-02-04T14:02:44.747Z 23 460-479 48 50 50 76 77 77 F F F <---
55662024-02-04T14:02:44.747Z 24 480-499 47 47 47 75 75 75 F F F
55672024-02-04T14:02:44.747Z 25 500-519 48 48 48 76 76 76 F F F
55682024-02-04T14:02:44.747Z 26 520-539 44 50 50 71 77 77 F F F <---
55692024-02-04T14:02:44.747Z 27 540-559 47 50 50 75 77 77 F F F <---
55702024-02-04T14:02:44.747Z 28 560-579 48 48 48 76 76 76 F T T <---
55712024-02-04T14:02:44.747Z 29 580-599 42 42 42 69 69 69 F T T <---
55722024-02-04T14:02:44.747ZMax gen: 50, Max flush: 77
55732024-02-04T14:02:44.747ZError: Difference in extent metadata found!
55742024-02-04T14:02:44.750ZA Difference in extent metadata is expected here
55752024-02-04T14:02:44.750ZOn loop 25
55762024-02-04T14:02:44.750Z
55772024-02-04T14:02:44.756ZVerifying data now
55782024-02-04T14:02:50.289ZNo change after verify
55792024-02-04T14:02:50.289ZLoop: 25 Downstairs dump after verify (and repair):
55802024-02-04T14:02:50.313ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55812024-02-04T14:02:50.314Z 0 000-019 51 51 51 78 78 78 F F F
55822024-02-04T14:02:50.314Z 1 020-039 51 51 51 78 78 78 F F F
55832024-02-04T14:02:50.314Z 2 040-059 48 48 48 76 76 76 F F F
55842024-02-04T14:02:50.314Z 3 060-079 45 45 45 73 73 73 F F F
55852024-02-04T14:02:50.314Z 4 080-099 35 35 35 58 58 58 F F F
55862024-02-04T14:02:50.314Z 5 100-119 51 51 51 78 78 78 F F F
55872024-02-04T14:02:50.314Z 6 120-139 51 51 51 78 78 78 F F F
55882024-02-04T14:02:50.314Z 7 140-159 48 48 48 76 76 76 F F F
55892024-02-04T14:02:50.317Z 8 160-179 51 51 51 78 78 78 F F F
55902024-02-04T14:02:50.317Z 9 180-199 48 48 48 76 76 76 F F F
55912024-02-04T14:02:50.317Z 10 200-219 48 48 48 76 76 76 F F F
55922024-02-04T14:02:50.317Z 11 220-239 40 40 40 67 67 67 F F F
55932024-02-04T14:02:50.317Z 12 240-259 50 50 50 77 77 77 F F F
55942024-02-04T14:02:50.317Z 13 260-279 44 44 44 72 72 72 F F F
55952024-02-04T14:02:50.318Z 14 280-299 45 45 45 73 73 73 F F F
55962024-02-04T14:02:50.318Z 15 300-319 51 51 51 78 78 78 F F F
55972024-02-04T14:02:50.318Z 16 320-339 51 51 51 78 78 78 F F F
55982024-02-04T14:02:50.318Z 17 340-359 51 51 51 78 78 78 F F F
55992024-02-04T14:02:50.318Z 18 360-379 50 50 50 77 77 77 F F F
56002024-02-04T14:02:50.318Z 19 380-399 50 50 50 77 77 77 F F F
56012024-02-04T14:02:50.318Z 20 400-419 50 50 50 77 77 77 F F F
56022024-02-04T14:02:50.318Z 21 420-439 50 50 50 77 77 77 F F F
56032024-02-04T14:02:50.318Z 22 440-459 47 47 47 75 75 75 F F F
56042024-02-04T14:02:50.318Z 23 460-479 50 50 50 77 77 77 F F F
56052024-02-04T14:02:50.318Z 24 480-499 47 47 47 75 75 75 F F F
56062024-02-04T14:02:50.318Z 25 500-519 48 48 48 76 76 76 F F F
56072024-02-04T14:02:50.318Z 26 520-539 50 50 50 77 77 77 F F F
56082024-02-04T14:02:50.318Z 27 540-559 50 50 50 77 77 77 F F F
56092024-02-04T14:02:50.318Z 28 560-579 51 51 51 78 78 78 F F F
56102024-02-04T14:02:50.318Z 29 580-599 51 51 51 78 78 78 F F F
56112024-02-04T14:02:50.318ZMax gen: 51, Max flush: 78
56122024-02-04T14:02:50.321Z
56132024-02-04T14:02:50.321ZBegin loop 26
56142024-02-04T14:02:50.321ZDownstairs to restart: 0
56152024-02-04T14:02:50.353Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.385826185Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337}
56162024-02-04T14:02:50.353Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.386321982Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337}
56172024-02-04T14:02:50.353Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.386350408Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337}
56182024-02-04T14:02:50.353Z{"msg":"Crucible 2b3e8601-9d1c-43e4-a4c0-45b1120c474d has session id: 7308431d-092c-4543-b109-da61a1fb6ac4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.386371938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56192024-02-04T14:02:50.356Z{"msg":"Upstairs opts: Upstairs UUID: 2b3e8601-9d1c-43e4-a4c0-45b1120c474d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.386477691Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337Crucible runtime is spawned
56202024-02-04T14:02:50.356Z,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56212024-02-04T14:02:50.356Z{"msg":"Crucible stats registered with UUID: 2b3e8601-9d1c-43e4-a4c0-45b1120c474d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.386599646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56222024-02-04T14:02:50.357Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.386655621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337}
56232024-02-04T14:02:50.357Z{"msg":"2b3e8601-9d1c-43e4-a4c0-45b1120c474d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387270895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56242024-02-04T14:02:50.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387342127Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56252024-02-04T14:02:50.357Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387369532Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56262024-02-04T14:02:50.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387412072Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56272024-02-04T14:02:50.357Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387436947Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56282024-02-04T14:02:50.357Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387501814Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56292024-02-04T14:02:50.357Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387547013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56302024-02-04T14:02:50.357Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.38760967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56312024-02-04T14:02:50.357Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387666187Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56322024-02-04T14:02:50.357Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.387760803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56332024-02-04T14:02:50.357Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.388023527Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56342024-02-04T14:02:50.357Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.388081663Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56352024-02-04T14:02:50.357Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.388125687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56362024-02-04T14:02:50.357Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.389281763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56372024-02-04T14:02:50.357Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.389340934Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56382024-02-04T14:02:50.357Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.389434078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56392024-02-04T14:02:50.357Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.389504923Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56402024-02-04T14:02:50.361Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.390120242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56412024-02-04T14:02:50.361Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.390328568Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56422024-02-04T14:02:50.361Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.390904622Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56432024-02-04T14:02:50.362Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.39096437Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56442024-02-04T14:02:50.362Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.390989863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56452024-02-04T14:02:50.362Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.391209657Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56462024-02-04T14:02:50.362Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.391274098Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56472024-02-04T14:02:50.362Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.39130075Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56482024-02-04T14:02:50.362Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.39132239Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56492024-02-04T14:02:50.362Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.391344906Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56502024-02-04T14:02:50.362Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.392126063Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56512024-02-04T14:02:50.362Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.392819861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56522024-02-04T14:02:50.362Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.39290576Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56532024-02-04T14:02:50.362Z{"msg":"[0]R flush_numbers[0..12]: [78, 78, 76, 73, 58, 78, 78, 76, 78, 76, 76, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.392931019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56542024-02-04T14:02:50.362Z{"msg":"[0]R generation[0..12]: [51, 51, 48, 45, 35, 51, 51, 48, 51, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.392955008Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56552024-02-04T14:02:50.362Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.392979331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56562024-02-04T14:02:50.362Z{"msg":"[1]R flush_numbers[0..12]: [78, 78, 76, 73, 58, 78, 78, 76, 78, 76, 76, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.39302574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56572024-02-04T14:02:50.362Z{"msg":"[1]R generation[0..12]: [51, 51, 48, 45, 35, 51, 51, 48, 51, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393051742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56582024-02-04T14:02:50.362Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393076496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
56592024-02-04T14:02:50.362Z,"":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56602024-02-04T14:02:50.362Z{"msg":"[2]R flush_numbers[0..12]: [78, 78, 76, 73, 58, 78, 78, 76, 78, 76, 76, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393103453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56612024-02-04T14:02:50.362Z{"msg":"[2]R generation[0..12]: [51, 51, 48, 45, 35, 51, 51, 48, 51, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393126741Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56622024-02-04T14:02:50.362Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393152166Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56632024-02-04T14:02:50.362Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.39320478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56642024-02-04T14:02:50.362Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393229015Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56652024-02-04T14:02:50.363Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393273311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56662024-02-04T14:02:50.363Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393296765Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56672024-02-04T14:02:50.363Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393338902Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56682024-02-04T14:02:50.363Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393360658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56692024-02-04T14:02:50.363Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393401943Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56702024-02-04T14:02:50.363Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393425558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56712024-02-04T14:02:50.363Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393448204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56722024-02-04T14:02:50.363Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393468803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56732024-02-04T14:02:50.363Z{"msg":"2b3e8601-9d1c-43e4-a4c0-45b1120c474d is now active with session: 7308431d-092c-4543-b109-da61a1fb6ac4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393519616Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56742024-02-04T14:02:50.363Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393541485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337,"session_id":"7308431d-092c-4543-b109-da61a1fb6ac4"}
56752024-02-04T14:02:50.363Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:49.393580774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337}
56762024-02-04T14:02:50.363ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56772024-02-04T14:02:50.363ZLoading write count information from file "/tmp/test_repair_verify.data"
56782024-02-04T14:02:50.363ZRun Repair workload
56792024-02-04T14:02:50.369Z01/30 Write block 183 len 4096 data: 7
56802024-02-04T14:02:50.369Z02/30 Read block 436 len 32768
56812024-02-04T14:02:50.369Z03/30 Read block 386 len 24576
56822024-02-04T14:02:50.372Z04/30 Flush
56832024-02-04T14:02:50.372Z05/30 Write block 423 len 4096 data: 7
56842024-02-04T14:02:50.372Z06/30 Write block 232 len 40960 data: 3 4 4 3 3 4 4 4 5 5
56852024-02-04T14:02:50.372Z07/30 Write block 352 len 20480 data: 4 6 5 6 7
56862024-02-04T14:02:50.375Z08/30 Flush
56872024-02-04T14:02:50.378Z09/30 Write block 223 len 28672 data: 5 4 4 4 5 5 4
56882024-02-04T14:02:50.378Z10/30 Read block 469 len 8192
56892024-02-04T14:02:50.381Z11/30 Write block 361 len 28672 data: 6 6 7 7 7 4 5
56902024-02-04T14:02:50.381Z12/30 Write block 573 len 32768 data: 5 6 6 7 7 6 9 10
56912024-02-04T14:02:50.384Z13/30 Read block 529 len 40960
56922024-02-04T14:02:50.387Z14/30 Write block 313 len 8192 data: 5 4
56932024-02-04T14:02:50.387Z15/30 Write block 576 len 12288 data: 8 8 7
56942024-02-04T14:02:50.390Z16/30 Write block 454 len 40960 data: 6 5 5 5 5 4 4 6 5 5
56952024-02-04T14:02:50.390Z17/30 Read block 337 len 4096
56962024-02-04T14:02:50.390Z18/30 Read block 220 len 4096
56972024-02-04T14:02:50.392Z19/30 Read block 435 len 24576
56982024-02-04T14:02:50.392Z20/30 Read block 450 len 20480
56992024-02-04T14:02:50.395Z21/30 Write block 580 len 4096 data: 11
57002024-02-04T14:02:50.395Z22/30 Read block 230 len 36864
57012024-02-04T14:02:50.398Z23/30 Write block 221 len 8192 data: 5 4
57022024-02-04T14:02:50.398Z24/30 Read block 595 len 4096
57032024-02-04T14:02:50.398Z25/30 Read block 521 len 8192
57042024-02-04T14:02:50.401Z26/30 Flush
57052024-02-04T14:02:50.404Z27/30 Write block 44 len 4096 data: 4
57062024-02-04T14:02:50.404Z28/30 Write block 95 len 20480 data: 4 4 4 4 5
57072024-02-04T14:02:50.407Z29/30 Write block 108 len 36864 data: 9 8 8 9 11 9 9 9 9
57082024-02-04T14:02:50.407Z30/30 Write block 457 len 4096 data: 6
57092024-02-04T14:02:50.407Z----------------------------------------------------------------
57102024-02-04T14:02:50.407Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57112024-02-04T14:02:50.407ZStates: Active Active Active
57122024-02-04T14:02:50.407ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57132024-02-04T14:02:50.407Z 1 Acked 1000 Write 1 Sent Done Done false
57142024-02-04T14:02:50.407Z 2 Acked 1001 Read 8 Sent Done Done false
57152024-02-04T14:02:50.407Z 3 Acked 1002 Read 6 Sent Done Done false
57162024-02-04T14:02:50.407Z 4 Acked 1003 Flush 0 Sent Done Done false
57172024-02-04T14:02:50.407Z 5 Acked 1004 Write 1 Sent Done Done false
57182024-02-04T14:02:50.407Z 6 Acked 1005 Write 10 Sent Done Done false
57192024-02-04T14:02:50.407Z 7 Acked 1006 Write 5 Sent Done Done false
57202024-02-04T14:02:50.408Z 8 Acked 1007 Flush 0 Sent Done Done false
57212024-02-04T14:02:50.408Z 9 Acked 1008 Write 7 Sent Done Done false
57222024-02-04T14:02:50.408Z 10 Acked 1009 Read 2 Sent Done Done false
57232024-02-04T14:02:50.408Z 11 Acked 1010 Write 7 Sent Done Done false
57242024-02-04T14:02:50.408Z 12 Acked 1011 Write 8 Sent Done Done false
57252024-02-04T14:02:50.408Z 13 Acked 1012 Read 10 Sent Done Done false
57262024-02-04T14:02:50.408Z 14 Acked 1013 Write 2 Sent Done Done false
57272024-02-04T14:02:50.408Z 15 Acked 1014 Write 3 Sent Done Done false
57282024-02-04T14:02:50.408Z 16 Acked 1015 Write 10 Sent Done Done false
57292024-02-04T14:02:50.408Z 17 Acked 1016 Read 1 Sent Done Done false
57302024-02-04T14:02:50.408Z 18 Acked 1017 Read 1 Sent Done Done false
57312024-02-04T14:02:50.408Z 19 Acked 1018 Read 6 Sent Done Done false
57322024-02-04T14:02:50.408Z 20 Acked 1019 Read 5 Sent Done Done false
57332024-02-04T14:02:50.408Z 21 Acked 1020 Write 1 Sent Done Done false
57342024-02-04T14:02:50.408Z 22 Acked 1021 Read 9 Sent Done Done false
57352024-02-04T14:02:50.408Z 23 Acked 1022 Write 2 Sent Done Done false
57362024-02-04T14:02:50.408Z 24 Acked 1023 Read 1 Sent Done Done false
57372024-02-04T14:02:50.408Z 25 Acked 1024 Read 2 Sent Done Done false
57382024-02-04T14:02:50.408Z 26 Acked 1025 Flush 0 Sent Done Done false
57392024-02-04T14:02:50.408Z 27 Acked 1026 Write 1 Sent Done Done false
57402024-02-04T14:02:50.408Z 28 Acked 1027 Write 5 Sent Done Done false
57412024-02-04T14:02:50.408Z 29 Acked 1028 Write 9 Sent Sent Sent false
57422024-02-04T14:02:50.408Z 30 Acked 1029 Write 1 Sent Sent Sent false
57432024-02-04T14:02:50.408Z STATES DS:0 DS:1 DS:2 TOTAL
57442024-02-04T14:02:50.409Z New 0 0 0 0
57452024-02-04T14:02:50.409Z Sent 30 2 2 34
57462024-02-04T14:02:50.409Z Done 0 28 28 56
57472024-02-04T14:02:50.409Z Skipped 0 0 0 0
57482024-02-04T14:02:50.409Z Error 0 0 0 0
57492024-02-04T14:02:50.409ZLast Flush: 0 1025 1025
57502024-02-04T14:02:50.409ZDownstairs last five completed:
57512024-02-04T14:02:50.409ZUpstairs last five completed: 30 29 28 27 26
57522024-02-04T14:02:50.409Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:49.440310205Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1337}
57532024-02-04T14:02:50.412ZWrote out file "/tmp/test_repair_verify.data"
57542024-02-04T14:02:50.415Z
57552024-02-04T14:02:50.420ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57562024-02-04T14:02:50.454ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57572024-02-04T14:02:50.454Z 0 000-019 51 51 51 78 78 78 F F F
57582024-02-04T14:02:50.454Z 1 020-039 51 51 51 78 78 78 F F F
57592024-02-04T14:02:50.455Z 2 040-059 48 48 48 76 76 76 F T T <---
57602024-02-04T14:02:50.455Z 3 060-079 45 45 45 73 73 73 F F F
57612024-02-04T14:02:50.455Z 4 080-099 35 35 35 58 58 58 F T T <---
57622024-02-04T14:02:50.455Z 5 100-119 51 51 51 78 78 78 F T T <---
57632024-02-04T14:02:50.455Z 6 120-139 51 51 51 78 78 78 F F F
57642024-02-04T14:02:50.455Z 7 140-159 48 48 48 76 76 76 F F F
57652024-02-04T14:02:50.455Z 8 160-179 51 51 51 78 78 78 F F F
57662024-02-04T14:02:50.458Z 9 180-199 48 52 52 76 79 79 F F F <---
57672024-02-04T14:02:50.459Z 10 200-219 48 48 48 76 76 76 F F F
57682024-02-04T14:02:50.459Z 11 220-239 40 52 52 67 81 81 F F F <---
57692024-02-04T14:02:50.459Z 12 240-259 50 52 52 77 80 80 F F F <---
57702024-02-04T14:02:50.459Z 13 260-279 44 44 44 72 72 72 F F F
57712024-02-04T14:02:50.459Z 14 280-299 45 45 45 73 73 73 F F F
57722024-02-04T14:02:50.459Z 15 300-319 51 52 52 78 81 81 F F F <---
57732024-02-04T14:02:50.459Z 16 320-339 51 51 51 78 78 78 F F F
57742024-02-04T14:02:50.459Z 17 340-359 51 52 52 78 80 80 F F F <---
57752024-02-04T14:02:50.459Z 18 360-379 50 52 52 77 81 81 F F F <---
57762024-02-04T14:02:50.459Z 19 380-399 50 50 50 77 77 77 F F F
57772024-02-04T14:02:50.459Z 20 400-419 50 50 50 77 77 77 F F F
57782024-02-04T14:02:50.459Z 21 420-439 50 52 52 77 80 80 F F F <---
57792024-02-04T14:02:50.459Z 22 440-459 47 52 52 75 81 81 F T T <---
57802024-02-04T14:02:50.459Z 23 460-479 50 52 52 77 81 81 F F F <---
57812024-02-04T14:02:50.460Z 24 480-499 47 47 47 75 75 75 F F F
57822024-02-04T14:02:50.460Z 25 500-519 48 48 48 76 76 76 F F F
57832024-02-04T14:02:50.460Z 26 520-539 50 50 50 77 77 77 F F F
57842024-02-04T14:02:50.460Z 27 540-559 50 50 50 77 77 77 F F F
57852024-02-04T14:02:50.460Z 28 560-579 51 52 52 78 81 81 F F F <---
57862024-02-04T14:02:50.460Z 29 580-599 51 52 52 78 81 81 F F F <---
57872024-02-04T14:02:50.460ZMax gen: 52, Max flush: 81
57882024-02-04T14:02:50.460ZError: Difference in extent metadata found!
57892024-02-04T14:02:50.460ZA Difference in extent metadata is expected here
57902024-02-04T14:02:50.460ZOn loop 26
57912024-02-04T14:02:50.460Z
57922024-02-04T14:02:50.460ZVerifying data now
57932024-02-04T14:02:55.669ZNo change after verify
57942024-02-04T14:02:55.669ZLoop: 26 Downstairs dump after verify (and repair):
57952024-02-04T14:02:55.691ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57962024-02-04T14:02:55.691Z 0 000-019 51 51 51 78 78 78 F F F
57972024-02-04T14:02:55.691Z 1 020-039 51 51 51 78 78 78 F F F
57982024-02-04T14:02:55.691Z 2 040-059 53 53 53 82 82 82 F F F
57992024-02-04T14:02:55.692Z 3 060-079 45 45 45 73 73 73 F F F
58002024-02-04T14:02:55.692Z 4 080-099 53 53 53 82 82 82 F F F
58012024-02-04T14:02:55.692Z 5 100-119 53 53 53 82 82 82 F F F
58022024-02-04T14:02:55.695Z 6 120-139 51 51 51 78 78 78 F F F
58032024-02-04T14:02:55.695Z 7 140-159 48 48 48 76 76 76 F F F
58042024-02-04T14:02:55.695Z 8 160-179 51 51 51 78 78 78 F F F
58052024-02-04T14:02:55.695Z 9 180-199 52 52 52 79 79 79 F F F
58062024-02-04T14:02:55.695Z 10 200-219 48 48 48 76 76 76 F F F
58072024-02-04T14:02:55.695Z 11 220-239 52 52 52 81 81 81 F F F
58082024-02-04T14:02:55.695Z 12 240-259 52 52 52 80 80 80 F F F
58092024-02-04T14:02:55.695Z 13 260-279 44 44 44 72 72 72 F F F
58102024-02-04T14:02:55.695Z 14 280-299 45 45 45 73 73 73 F F F
58112024-02-04T14:02:55.695Z 15 300-319 52 52 52 81 81 81 F F F
58122024-02-04T14:02:55.695Z 16 320-339 51 51 51 78 78 78 F F F
58132024-02-04T14:02:55.695Z 17 340-359 52 52 52 80 80 80 F F F
58142024-02-04T14:02:55.695Z 18 360-379 52 52 52 81 81 81 F F F
58152024-02-04T14:02:55.695Z 19 380-399 50 50 50 77 77 77 F F F
58162024-02-04T14:02:55.695Z 20 400-419 50 50 50 77 77 77 F F F
58172024-02-04T14:02:55.696Z 21 420-439 52 52 52 80 80 80 F F F
58182024-02-04T14:02:55.696Z 22 440-459 53 53 53 82 82 82 F F F
58192024-02-04T14:02:55.696Z 23 460-479 52 52 52 81 81 81 F F F
58202024-02-04T14:02:55.696Z 24 480-499 47 47 47 75 75 75 F F F
58212024-02-04T14:02:55.696Z 25 500-519 48 48 48 76 76 76 F F F
58222024-02-04T14:02:55.696Z 26 520-539 50 50 50 77 77 77 F F F
58232024-02-04T14:02:55.696Z 27 540-559 50 50 50 77 77 77 F F F
58242024-02-04T14:02:55.696Z 28 560-579 52 52 52 81 81 81 F F F
58252024-02-04T14:02:55.696Z 29 580-599 52 52 52 81 81 81 F F F
58262024-02-04T14:02:55.696ZMax gen: 53, Max flush: 82
58272024-02-04T14:02:55.699Z
58282024-02-04T14:02:55.699ZBegin loop 27
58292024-02-04T14:02:55.699ZDownstairs to restart: 2
58302024-02-04T14:02:55.727Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.760238282Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345}
58312024-02-04T14:02:55.727Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.760724433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345}
58322024-02-04T14:02:55.727Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
58332024-02-04T14:02:55.733Z,"time":"2024-02-04T14:02:54.76075831Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345}
58342024-02-04T14:02:55.733Z{"msg":"Crucible a3c7bff1-10a1-4212-a076-375ee167f15b has session id: ce28fef1-133a-4e4b-9337-cce69b3e055e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.760931395Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58352024-02-04T14:02:55.733Z{"msg":"Upstairs opts: Upstairs UUID: a3c7bff1-10a1-4212-a076-375ee167f15b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761018676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58362024-02-04T14:02:55.733Z{"msg":"Crucible stats registered with UUID: a3c7bff1-10a1-4212-a076-375ee167f15b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761074446Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58372024-02-04T14:02:55.733Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761097974Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345}
58382024-02-04T14:02:55.734Z{"msg":"a3c7bff1-10a1-4212-a076-375ee167f15b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761273848Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58392024-02-04T14:02:55.734Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761323921Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58402024-02-04T14:02:55.734Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761353722Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58412024-02-04T14:02:55.734Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761420106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58422024-02-04T14:02:55.734Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761463823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58432024-02-04T14:02:55.734Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761539832Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58442024-02-04T14:02:55.734Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761586022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58452024-02-04T14:02:55.734Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761745601Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"io task","client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58462024-02-04T14:02:55.734Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.761804708Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"io task","client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58472024-02-04T14:02:55.734Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.762107559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"io task","client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58482024-02-04T14:02:55.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.762157288Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"io task","client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58492024-02-04T14:02:55.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.76223566Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"io task","client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58502024-02-04T14:02:55.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.762284549Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"io task","client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58512024-02-04T14:02:55.734Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.763168674Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58522024-02-04T14:02:55.734Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.763254346Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58532024-02-04T14:02:55.734Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.763300027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58542024-02-04T14:02:55.734Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.763369837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58552024-02-04T14:02:55.738Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.764074658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58562024-02-04T14:02:55.738Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.764372069Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58572024-02-04T14:02:55.738Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.76445183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58582024-02-04T14:02:55.739Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.764512503Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58592024-02-04T14:02:55.739Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.764858088Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58602024-02-04T14:02:55.739Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.764941334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58612024-02-04T14:02:55.739Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.764988134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58622024-02-04T14:02:55.739Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.765299383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58632024-02-04T14:02:55.739Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.765356576Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58642024-02-04T14:02:55.739Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.765381284Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58652024-02-04T14:02:55.739Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.765422204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58662024-02-04T14:02:55.739Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766125902Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58672024-02-04T14:02:55.739Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766241836Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58682024-02-04T14:02:55.739Z{"msg":"[0]R flush_numbers[0..12]: [78, 78, 82, 73, 82, 82, 78, 76, 78, 79, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766311614Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58692024-02-04T14:02:55.739Z{"msg":"[0]R generation[0..12]: [51, 51, 53, 45, 53, 53, 51, 48, 51, 52, 48, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.76633973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58702024-02-04T14:02:55.739Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766364682Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58712024-02-04T14:02:55.739Z{"msg":"[1]R flush_numbers[0..12]: [78, 78, 82, 73, 82, 82, 78, 76, 78, 79, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766388823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58722024-02-04T14:02:55.739Z{"msg":"[1]R generation[0..12]: [51, 51, 53, 45, 53, 53, 51, 48, 51, 52, 48, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766412039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58732024-02-04T14:02:55.739Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766456573Z","hostname":"Wait for a query_work_queue command to finish before sending IO
58742024-02-04T14:02:55.739Zip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58752024-02-04T14:02:55.739Z{"msg":"[2]R flush_numbers[0..12]: [78, 78, 82, 73, 82, 82, 78, 76, 78, 79, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766487134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58762024-02-04T14:02:55.739Z{"msg":"[2]R generation[0..12]: [51, 51, 53, 45, 53, 53, 51, 48, 51, 52, 48, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.7665294Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58772024-02-04T14:02:55.739Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766554699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58782024-02-04T14:02:55.739Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766595462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58792024-02-04T14:02:55.739Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766618964Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58802024-02-04T14:02:55.739Z{"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766659253Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58812024-02-04T14:02:55.740Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766682401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58822024-02-04T14:02:55.740Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766704849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58832024-02-04T14:02:55.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766725079Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58842024-02-04T14:02:55.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766747312Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58852024-02-04T14:02:55.740Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766787854Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58862024-02-04T14:02:55.740Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766811473Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58872024-02-04T14:02:55.740Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766832615Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58882024-02-04T14:02:55.740Z{"msg":"a3c7bff1-10a1-4212-a076-375ee167f15b is now active with session: ce28fef1-133a-4e4b-9337-cce69b3e055e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766878789Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58892024-02-04T14:02:55.740Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766900056Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345,"session_id":"ce28fef1-133a-4e4b-9337-cce69b3e055e"}
58902024-02-04T14:02:55.740Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:54.766921906Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345}
58912024-02-04T14:02:55.740ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58922024-02-04T14:02:55.740ZLoading write count information from file "/tmp/test_repair_verify.data"
58932024-02-04T14:02:55.740ZRun Repair workload
58942024-02-04T14:02:55.740Z01/30 Write block 54 len 8192 data: 2 2
58952024-02-04T14:02:55.740Z02/30 Read block 256 len 12288
58962024-02-04T14:02:55.743Z03/30 Flush
58972024-02-04T14:02:55.743Z04/30 Write block 351 len 24576 data: 4 5 7 6 7 8
58982024-02-04T14:02:55.743Z05/30 Write block 180 len 16384 data: 7 7 6 8
58992024-02-04T14:02:55.744Z06/30 Write block 412 len 4096 data: 5
59002024-02-04T14:02:55.744Z07/30 Flush
59012024-02-04T14:02:55.746Z08/30 Flush
59022024-02-04T14:02:55.749Z09/30 Flush
59032024-02-04T14:02:55.749Z10/30 Write block 72 len 20480 data: 3 3 4 4 4
59042024-02-04T14:02:55.752Z11/30 Write block 15 len 40960 data: 4 4 3 3 3 3 2 2 2 3
59052024-02-04T14:02:55.752Z12/30 Read block 496 len 4096
59062024-02-04T14:02:55.755Z13/30 Write block 161 len 28672 data: 5 5 5 5 6 6 6
59072024-02-04T14:02:55.755Z14/30 Write block 423 len 20480 data: 8 4 5 5 4
59082024-02-04T14:02:55.758Z15/30 Read block 537 len 40960
59092024-02-04T14:02:55.758Z16/30 Read block 363 len 28672
59102024-02-04T14:02:55.760Z17/30 Write block 478 len 8192 data: 6 6
59112024-02-04T14:02:55.761Z18/30 Read block 237 len 4096
59122024-02-04T14:02:55.763Z19/30 Read block 556 len 16384
59132024-02-04T14:02:55.763Z20/30 Read block 82 len 16384
59142024-02-04T14:02:55.766Z21/30 Write block 284 len 4096 data: 8
59152024-02-04T14:02:55.766Z22/30 Read block 269 len 24576
59162024-02-04T14:02:55.766Z23/30 Read block 366 len 40960
59172024-02-04T14:02:55.769Z24/30 Flush
59182024-02-04T14:02:55.772Z25/30 Write block 147 len 32768 data: 4 4 4 4 4 4 4 3
59192024-02-04T14:02:55.775Z26/30 Read block 303 len 24576
59202024-02-04T14:02:55.775Z27/30 Read block 133 len 24576
59212024-02-04T14:02:55.778Z28/30 Read block 311 len 4096
59222024-02-04T14:02:55.778Z29/30 Write block 416 len 28672 data: 5 5 4 5 8 8 7
59232024-02-04T14:02:55.781Z30/30 Read block 332 len 24576
59242024-02-04T14:02:55.781Z----------------------------------------------------------------
59252024-02-04T14:02:55.781Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:27
59262024-02-04T14:02:55.781ZStates: Active Active Active
59272024-02-04T14:02:55.781ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59282024-02-04T14:02:55.781Z 4 Acked 1003 Write 6 Done Done Done false
59292024-02-04T14:02:55.781Z 5 Acked 1004 Write 4 Done Done Sent false
59302024-02-04T14:02:55.781Z 6 Acked 1005 Write 1 Done Done Sent false
59312024-02-04T14:02:55.781Z 7 Acked 1006 Flush 0 Done Done Sent false
59322024-02-04T14:02:55.781Z 8 Acked 1007 Flush 0 Done Done Sent false
59332024-02-04T14:02:55.781Z 9 Acked 1008 Flush 0 Done Done Sent false
59342024-02-04T14:02:55.781Z 10 Acked 1009 Write 5 Done Done Sent false
59352024-02-04T14:02:55.781Z 11 Acked 1010 Write 10 Done Done Sent false
59362024-02-04T14:02:55.781Z 12 Acked 1011 Read 1 Done Done Sent false
59372024-02-04T14:02:55.781Z 13 Acked 1012 Write 7 Done Done Sent false
59382024-02-04T14:02:55.782Z 14 Acked 1013 Write 5 Done Done Sent false
59392024-02-04T14:02:55.782Z 15 Acked 1014 Read 10 Done Done Sent false
59402024-02-04T14:02:55.782Z 16 Acked 1015 Read 7 Done Done Sent false
59412024-02-04T14:02:55.782Z 17 Acked 1016 Write 2 Done Done Sent false
59422024-02-04T14:02:55.782Z 18 Acked 1017 Read 1 Done Done Sent false
59432024-02-04T14:02:55.782Z 19 Acked 1018 Read 4 Done Done Sent false
59442024-02-04T14:02:55.782Z 20 Acked 1019 Read 4 Done Done Sent false
59452024-02-04T14:02:55.782Z 21 Acked 1020 Write 1 Done Done Sent false
59462024-02-04T14:02:55.782Z 22 Acked 1021 Read 6 Done Done Sent false
59472024-02-04T14:02:55.782Z 23 Acked 1022 Read 10 Done Done Sent false
59482024-02-04T14:02:55.782Z 24 Acked 1023 Flush 0 Done Done Sent false
59492024-02-04T14:02:55.782Z 25 Acked 1024 Write 8 Done Done Sent false
59502024-02-04T14:02:55.782Z 26 Acked 1025 Read 6 Done Done Sent false
59512024-02-04T14:02:55.782Z 27 Acked 1026 Read 6 Done Done Sent false
59522024-02-04T14:02:55.782Z 28 Acked 1027 Read 1 Done Done Sent false
59532024-02-04T14:02:55.782Z 29 Acked 1028 Write 7 Done Done Sent false
59542024-02-04T14:02:55.782Z 30 Acked 1029 Read 6 Done Done Sent false
59552024-02-04T14:02:55.782Z STATES DS:0 DS:1 DS:2 TOTAL
59562024-02-04T14:02:55.782Z New 0 0 0 0
59572024-02-04T14:02:55.782Z Sent 0 0 26 26
59582024-02-04T14:02:55.782Z Done 27 27 1 55
59592024-02-04T14:02:55.782Z Skipped 0 0 0 0
59602024-02-04T14:02:55.782Z Error 0 0 0 0
59612024-02-04T14:02:55.782ZLast Flush: 1023 1023 1002
59622024-02-04T14:02:55.783ZDownstairs last five completed: 1002 1001 1000
59632024-02-04T14:02:55.783ZUpstairs last five completed: 30 29 28 27 26
59642024-02-04T14:02:55.786Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:54.814544746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1345}
59652024-02-04T14:02:55.786ZWrote out file "/tmp/test_repair_verify.data"
59662024-02-04T14:02:55.789Z
59672024-02-04T14:02:55.796ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59682024-02-04T14:02:55.817ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59692024-02-04T14:02:55.818Z 0 000-019 54 54 51 87 87 78 F F F <---
59702024-02-04T14:02:55.818Z 1 020-039 54 54 51 87 87 78 F F F <---
59712024-02-04T14:02:55.818Z 2 040-059 54 54 54 83 83 83 F F F
59722024-02-04T14:02:55.818Z 3 060-079 54 54 45 87 87 73 F F F <---
59732024-02-04T14:02:55.818Z 4 080-099 53 53 53 82 82 82 F F F
59742024-02-04T14:02:55.818Z 5 100-119 53 53 53 82 82 82 F F F
59752024-02-04T14:02:55.821Z 6 120-139 51 51 51 78 78 78 F F F
59762024-02-04T14:02:55.821Z 7 140-159 48 48 48 76 76 76 T T F <---
59772024-02-04T14:02:55.821Z 8 160-179 54 54 51 87 87 78 F F F <---
59782024-02-04T14:02:55.821Z 9 180-199 54 54 52 84 84 79 F F F <---
59792024-02-04T14:02:55.821Z 10 200-219 48 48 48 76 76 76 F F F
59802024-02-04T14:02:55.821Z 11 220-239 52 52 52 81 81 81 F F F
59812024-02-04T14:02:55.821Z 12 240-259 52 52 52 80 80 80 F F F
59822024-02-04T14:02:55.821Z 13 260-279 44 44 44 72 72 72 F F F
59832024-02-04T14:02:55.821Z 14 280-299 54 54 45 87 87 73 F F F <---
59842024-02-04T14:02:55.821Z 15 300-319 52 52 52 81 81 81 F F F
59852024-02-04T14:02:55.821Z 16 320-339 51 51 51 78 78 78 F F F
59862024-02-04T14:02:55.821Z 17 340-359 54 54 52 84 84 80 F F T <---
59872024-02-04T14:02:55.822Z 18 360-379 52 52 52 81 81 81 F F F
59882024-02-04T14:02:55.822Z 19 380-399 50 50 50 77 77 77 F F F
59892024-02-04T14:02:55.822Z 20 400-419 54 54 50 84 84 77 T T F <---
59902024-02-04T14:02:55.822Z 21 420-439 54 54 52 87 87 80 T T F <---
59912024-02-04T14:02:55.822Z 22 440-459 53 53 53 82 82 82 F F F
59922024-02-04T14:02:55.822Z 23 460-479 54 54 52 87 87 81 F F F <---
59932024-02-04T14:02:55.822Z 24 480-499 47 47 47 75 75 75 F F F
59942024-02-04T14:02:55.822Z 25 500-519 48 48 48 76 76 76 F F F
59952024-02-04T14:02:55.822Z 26 520-539 50 50 50 77 77 77 F F F
59962024-02-04T14:02:55.822Z 27 540-559 50 50 50 77 77 77 F F F
59972024-02-04T14:02:55.822Z 28 560-579 52 52 52 81 81 81 F F F
59982024-02-04T14:02:55.822Z 29 580-599 52 52 52 81 81 81 F F F
59992024-02-04T14:02:55.822ZMax gen: 54, Max flush: 87
60002024-02-04T14:02:55.822ZError: Difference in extent metadata found!
60012024-02-04T14:02:55.825ZA Difference in extent metadata is expected here
60022024-02-04T14:02:55.825ZOn loop 27
60032024-02-04T14:02:55.825Z
60042024-02-04T14:02:55.828ZVerifying data now
60052024-02-04T14:03:00.808ZNo change after verify
60062024-02-04T14:03:00.808ZLoop: 27 Downstairs dump after verify (and repair):
60072024-02-04T14:03:00.831ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60082024-02-04T14:03:00.831Z 0 000-019 54 54 54 87 87 87 F F F
60092024-02-04T14:03:00.831Z 1 020-039 54 54 54 87 87 87 F F F
60102024-02-04T14:03:00.831Z 2 040-059 54 54 54 83 83 83 F F F
60112024-02-04T14:03:00.831Z 3 060-079 54 54 54 87 87 87 F F F
60122024-02-04T14:03:00.831Z 4 080-099 53 53 53 82 82 82 F F F
60132024-02-04T14:03:00.834Z 5 100-119 53 53 53 82 82 82 F F F
60142024-02-04T14:03:00.834Z 6 120-139 51 51 51 78 78 78 F F F
60152024-02-04T14:03:00.834Z 7 140-159 55 55 55 88 88 88 F F F
60162024-02-04T14:03:00.834Z 8 160-179 54 54 54 87 87 87 F F F
60172024-02-04T14:03:00.834Z 9 180-199 54 54 54 84 84 84 F F F
60182024-02-04T14:03:00.834Z 10 200-219 48 48 48 76 76 76 F F F
60192024-02-04T14:03:00.834Z 11 220-239 52 52 52 81 81 81 F F F
60202024-02-04T14:03:00.834Z 12 240-259 52 52 52 80 80 80 F F F
60212024-02-04T14:03:00.834Z 13 260-279 44 44 44 72 72 72 F F F
60222024-02-04T14:03:00.834Z 14 280-299 54 54 54 87 87 87 F F F
60232024-02-04T14:03:00.834Z 15 300-319 52 52 52 81 81 81 F F F
60242024-02-04T14:03:00.834Z 16 320-339 51 51 51 78 78 78 F F F
60252024-02-04T14:03:00.834Z 17 340-359 54 54 54 84 84 84 F F F
60262024-02-04T14:03:00.834Z 18 360-379 52 52 52 81 81 81 F F F
60272024-02-04T14:03:00.834Z 19 380-399 50 50 50 77 77 77 F F F
60282024-02-04T14:03:00.835Z 20 400-419 55 55 55 88 88 88 F F F
60292024-02-04T14:03:00.835Z 21 420-439 55 55 55 88 88 88 F F F
60302024-02-04T14:03:00.835Z 22 440-459 53 53 53 82 82 82 F F F
60312024-02-04T14:03:00.835Z 23 460-479 54 54 54 87 87 87 F F F
60322024-02-04T14:03:00.835Z 24 480-499 47 47 47 75 75 75 F F F
60332024-02-04T14:03:00.835Z 25 500-519 48 48 48 76 76 76 F F F
60342024-02-04T14:03:00.835Z 26 520-539 50 50 50 77 77 77 F F F
60352024-02-04T14:03:00.835Z 27 540-559 50 50 50 77 77 77 F F F
60362024-02-04T14:03:00.835Z 28 560-579 52 52 52 81 81 81 F F F
60372024-02-04T14:03:00.835Z 29 580-599 52 52 52 81 81 81 F F F
60382024-02-04T14:03:00.835ZMax gen: 55, Max flush: 88
60392024-02-04T14:03:00.838Z
60402024-02-04T14:03:00.838ZBegin loop 28
60412024-02-04T14:03:00.838ZDownstairs to restart: 1
60422024-02-04T14:03:00.867Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.899746543Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353}
60432024-02-04T14:03:00.867Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900305153Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353}
60442024-02-04T14:03:00.871Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900352233Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353}
60452024-02-04T14:03:00.871Z{"msg":"Crucible a7366a47-12ae-4e2f-a96d-4bea669ed324 has session id: a31e31ec-e68e-418e-944c-3f6b1c58be4f","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900394201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_idCrucible runtime is spawned
60462024-02-04T14:03:00.871Z":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60472024-02-04T14:03:00.871Z{"msg":"Upstairs opts: Upstairs UUID: a7366a47-12ae-4e2f-a96d-4bea669ed324, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900469365Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60482024-02-04T14:03:00.871Z{"msg":"Crucible stats registered with UUID: a7366a47-12ae-4e2f-a96d-4bea669ed324","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900500183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60492024-02-04T14:03:00.871Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900562852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353}
60502024-02-04T14:03:00.871Z{"msg":"a7366a47-12ae-4e2f-a96d-4bea669ed324 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900664059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60512024-02-04T14:03:00.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900715682Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60522024-02-04T14:03:00.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900741217Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60532024-02-04T14:03:00.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900768466Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60542024-02-04T14:03:00.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900808929Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60552024-02-04T14:03:00.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.90084709Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60562024-02-04T14:03:00.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.900953482Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60572024-02-04T14:03:00.871Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.901002893Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"io task","client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60582024-02-04T14:03:00.871Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.901078045Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"io task","client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60592024-02-04T14:03:00.871Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.901129127Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"io task","client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60602024-02-04T14:03:00.872Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.901405642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"io task","client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60612024-02-04T14:03:00.872Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.901573543Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"io task","client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60622024-02-04T14:03:00.872Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.901663675Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"io task","client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60632024-02-04T14:03:00.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.902670392Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60642024-02-04T14:03:00.872Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.902784083Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60652024-02-04T14:03:00.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.902887186Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60662024-02-04T14:03:00.872Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.902918988Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60672024-02-04T14:03:00.872Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.90358905Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60682024-02-04T14:03:00.876Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.903735654Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60692024-02-04T14:03:00.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.904474283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60702024-02-04T14:03:00.876Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.904526946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60712024-02-04T14:03:00.876Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.904551728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60722024-02-04T14:03:00.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.904572796Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60732024-02-04T14:03:00.876Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.904618416Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60742024-02-04T14:03:00.876Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.904739159Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60752024-02-04T14:03:00.876Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.904766272Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60762024-02-04T14:03:00.876Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.904836309Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60772024-02-04T14:03:00.876Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.905577222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60782024-02-04T14:03:00.877Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906246675Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60792024-02-04T14:03:00.877Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906353167Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60802024-02-04T14:03:00.877Z{"msg":"[0]R flush_numbers[0..12]: [87, 87, 83, 87, 82, 82, 78, 88, 87, 84, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906379656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60812024-02-04T14:03:00.877Z{"msg":"[0]R generation[0..12]: [54, 54, 54, 54, 53, 53, 51, 55, 54, 54, 48, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906403686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60822024-02-04T14:03:00.877Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906447015Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60832024-02-04T14:03:00.877Z{"msg":"[1]R flush_numbers[0..12]: [87, 87, 83, 87, 82, 82, 78, 88, 87, 84, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906472478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60842024-02-04T14:03:00.877Z{"msg":"[1]R generation[0..12]: [54, 54, 54, 54, 53, 53, 51, 55, 54, 54, 48, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906514199Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60852024-02-04T14:03:00.877Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906539891Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60862024-02-04T14:03:00.877Z{"msg":"[2]R flush_numbers[0..12]: [87, 87, 83, 87, 82, 82, 78, 88, 87, 84, 76, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906621816Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60872024-02-04T14:03:00.877Z{"msg":"[2]R generation[0..12]: [54, 54, 54, 54, 53, 53, 51, 55, 54, 54, 48, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906646663Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60882024-02-04T14:03:00.877Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO
60892024-02-04T14:03:00.877Z:30,"time":"2024-02-04T14:02:59.906675982Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60902024-02-04T14:03:00.877Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906721781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60912024-02-04T14:03:00.877Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906745822Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60922024-02-04T14:03:00.877Z{"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906767586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60932024-02-04T14:03:00.877Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906823192Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60942024-02-04T14:03:00.877Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906847472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60952024-02-04T14:03:00.877Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906893375Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60962024-02-04T14:03:00.877Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906916873Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60972024-02-04T14:03:00.877Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.90693889Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60982024-02-04T14:03:00.877Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.906961661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
60992024-02-04T14:03:00.877Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.907001089Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
61002024-02-04T14:03:00.877Z{"msg":"a7366a47-12ae-4e2f-a96d-4bea669ed324 is now active with session: a31e31ec-e68e-418e-944c-3f6b1c58be4f","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.907025092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
61012024-02-04T14:03:00.877Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.907045743Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353,"session_id":"a31e31ec-e68e-418e-944c-3f6b1c58be4f"}
61022024-02-04T14:03:00.878Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:02:59.907066468Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353}
61032024-02-04T14:03:00.878ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61042024-02-04T14:03:00.878ZLoading write count information from file "/tmp/test_repair_verify.data"
61052024-02-04T14:03:00.878ZRun Repair workload
61062024-02-04T14:03:00.878Z01/30 Write block 232 len 8192 data: 4 5
61072024-02-04T14:03:00.882Z02/30 Read block 253 len 8192
61082024-02-04T14:03:00.882Z03/30 Write block 22 len 28672 data: 3 3 4 3 3 3 4
61092024-02-04T14:03:00.882Z04/30 Read block 159 len 8192
61102024-02-04T14:03:00.882Z05/30 Write block 341 len 4096 data: 3
61112024-02-04T14:03:00.885Z06/30 Write block 501 len 20480 data: 6 4 4 5 5
61122024-02-04T14:03:00.885Z07/30 Write block 485 len 20480 data: 4 4 4 5 7
61132024-02-04T14:03:00.885Z08/30 Write block 363 len 4096 data: 8
61142024-02-04T14:03:00.885Z09/30 Read block 382 len 8192
61152024-02-04T14:03:00.888Z10/30 Write block 67 len 24576 data: 4 4 3 3 3 4
61162024-02-04T14:03:00.888Z11/30 Read block 568 len 4096
61172024-02-04T14:03:00.888Z12/30 Flush
61182024-02-04T14:03:00.892Z13/30 Write block 140 len 8192 data: 4 3
61192024-02-04T14:03:00.895Z14/30 Read block 225 len 12288
61202024-02-04T14:03:00.895Z15/30 Flush
61212024-02-04T14:03:00.899Z16/30 Write block 498 len 40960 data: 7 7 7 7 5 5 6 6 7 7
61222024-02-04T14:03:00.899Z17/30 Write block 352 len 24576 data: 6 8 7 8 9 5
61232024-02-04T14:03:00.902Z18/30 Read block 306 len 8192
61242024-02-04T14:03:00.902Z19/30 Read block 269 len 32768
61252024-02-04T14:03:00.907Z20/30 Write block 349 len 16384 data: 5 3 5 7
61262024-02-04T14:03:00.907Z21/30 Write block 440 len 24576 data: 3 4 3 3 3 3
61272024-02-04T14:03:00.908Z22/30 Write block 399 len 4096 data: 6
61282024-02-04T14:03:00.911Z23/30 Read block 355 len 32768
61292024-02-04T14:03:00.911Z24/30 Write block 451 len 40960 data: 4 5 4 7 6 6 7 6 5 5
61302024-02-04T14:03:00.911Z25/30 Read block 65 len 36864
61312024-02-04T14:03:00.914Z26/30 Write block 475 len 8192 data: 8 6
61322024-02-04T14:03:00.914Z27/30 Write block 523 len 12288 data: 2 2 2
61332024-02-04T14:03:00.914Z28/30 Read block 319 len 16384
61342024-02-04T14:03:00.917Z29/30 Read block 522 len 24576
61352024-02-04T14:03:00.917Z30/30 Read block 312 len 20480
61362024-02-04T14:03:00.921Z----------------------------------------------------------------
61372024-02-04T14:03:00.921Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61382024-02-04T14:03:00.921ZStates: Active Active Active
61392024-02-04T14:03:00.921ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61402024-02-04T14:03:00.921Z 1 Acked 1000 Write 2 Done Sent Done false
61412024-02-04T14:03:00.921Z 2 Acked 1001 Read 2 Done Sent Done false
61422024-02-04T14:03:00.921Z 3 Acked 1002 Write 7 Done Sent Done false
61432024-02-04T14:03:00.921Z 4 Acked 1003 Read 2 Done Sent Done false
61442024-02-04T14:03:00.921Z 5 Acked 1004 Write 1 Done Sent Done false
61452024-02-04T14:03:00.921Z 6 Acked 1005 Write 5 Done Sent Done false
61462024-02-04T14:03:00.921Z 7 Acked 1006 Write 5 Done Sent Done false
61472024-02-04T14:03:00.922Z 8 Acked 1007 Write 1 Done Sent Done false
61482024-02-04T14:03:00.922Z 9 Acked 1008 Read 2 Done Sent Done false
61492024-02-04T14:03:00.922Z 10 Acked 1009 Write 6 Done Sent Done false
61502024-02-04T14:03:00.922Z 11 Acked 1010 Read 1 Done Sent Done false
61512024-02-04T14:03:00.922Z 12 Acked 1011 Flush 0 Done Sent Done false
61522024-02-04T14:03:00.922Z 13 Acked 1012 Write 2 Done Sent Done false
61532024-02-04T14:03:00.922Z 14 Acked 1013 Read 3 Done Sent Done false
61542024-02-04T14:03:00.922Z 15 Acked 1014 Flush 0 Done Sent Done false
61552024-02-04T14:03:00.922Z 16 Acked 1015 Write 10 Done Sent Done false
61562024-02-04T14:03:00.922Z 17 Acked 1016 Write 6 Done Sent Done false
61572024-02-04T14:03:00.922Z 18 Acked 1017 Read 2 Done Sent Done false
61582024-02-04T14:03:00.922Z 19 Acked 1018 Read 8 Done Sent Done false
61592024-02-04T14:03:00.922Z 20 Acked 1019 Write 4 Done Sent Done false
61602024-02-04T14:03:00.922Z 21 Acked 1020 Write 6 Done Sent Done false
61612024-02-04T14:03:00.922Z 22 Acked 1021 Write 1 Done Sent Done false
61622024-02-04T14:03:00.922Z 23 Acked 1022 Read 8 Done Sent Done false
61632024-02-04T14:03:00.922Z 24 Acked 1023 Write 10 Done Sent Done false
61642024-02-04T14:03:00.922Z 25 Acked 1024 Read 9 Done Sent Done false
61652024-02-04T14:03:00.922Z 26 Acked 1025 Write 2 Done Sent Done false
61662024-02-04T14:03:00.922Z 27 Acked 1026 Write 3 Done Sent Done false
61672024-02-04T14:03:00.922Z 28 Acked 1027 Read 4 Done Sent Done false
61682024-02-04T14:03:00.922Z 29 Acked 1028 Read 6 Done Sent Done false
61692024-02-04T14:03:00.922Z 30 Acked 1029 Read 5 Done Sent Done false
61702024-02-04T14:03:00.923Z STATES DS:0 DS:1 DS:2 TOTAL
61712024-02-04T14:03:00.923Z New 0 0 0 0
61722024-02-04T14:03:00.923Z Sent 0 30 0 30
61732024-02-04T14:03:00.923Z Done 30 0 30 60
61742024-02-04T14:03:00.923Z Skipped 0 0 0 0
61752024-02-04T14:03:00.923Z Error 0 0 0 0
61762024-02-04T14:03:00.923ZLast Flush: 1014 0 1014
61772024-02-04T14:03:00.923ZDownstairs last five completed:
61782024-02-04T14:03:00.923ZUpstairs last five completed: 30 29 28 27 26
61792024-02-04T14:03:00.923Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:02:59.952108287Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1353}
61802024-02-04T14:03:00.923ZWrote out file "/tmp/test_repair_verify.data"
61812024-02-04T14:03:00.927Z
61822024-02-04T14:03:00.932ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61832024-02-04T14:03:00.960ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61842024-02-04T14:03:00.960Z 0 000-019 54 54 54 87 87 87 F F F
61852024-02-04T14:03:00.960Z 1 020-039 56 54 56 89 87 89 F F F <---
61862024-02-04T14:03:00.960Z 2 040-059 54 54 54 83 83 83 F F F
61872024-02-04T14:03:00.960Z 3 060-079 56 54 56 89 87 89 F F F <---
61882024-02-04T14:03:00.960Z 4 080-099 53 53 53 82 82 82 F F F
61892024-02-04T14:03:00.960Z 5 100-119 53 53 53 82 82 82 F F F
61902024-02-04T14:03:00.960Z 6 120-139 51 51 51 78 78 78 F F F
61912024-02-04T14:03:00.960Z 7 140-159 56 55 56 90 88 90 F F F <---
61922024-02-04T14:03:00.960Z 8 160-179 54 54 54 87 87 87 F F F
61932024-02-04T14:03:00.960Z 9 180-199 54 54 54 84 84 84 F F F
61942024-02-04T14:03:00.963Z 10 200-219 48 48 48 76 76 76 F F F
61952024-02-04T14:03:00.964Z 11 220-239 56 52 56 89 81 89 F F F <---
61962024-02-04T14:03:00.964Z 12 240-259 52 52 52 80 80 80 F F F
61972024-02-04T14:03:00.964Z 13 260-279 44 44 44 72 72 72 F F F
61982024-02-04T14:03:00.964Z 14 280-299 54 54 54 87 87 87 F F F
61992024-02-04T14:03:00.964Z 15 300-319 52 52 52 81 81 81 F F F
62002024-02-04T14:03:00.964Z 16 320-339 51 51 51 78 78 78 F F F
62012024-02-04T14:03:00.964Z 17 340-359 56 54 56 89 84 89 T F T <---
62022024-02-04T14:03:00.964Z 18 360-379 56 52 56 89 81 89 F F F <---
62032024-02-04T14:03:00.964Z 19 380-399 50 50 50 77 77 77 T F T <---
62042024-02-04T14:03:00.964Z 20 400-419 55 55 55 88 88 88 F F F
62052024-02-04T14:03:00.964Z 21 420-439 55 55 55 88 88 88 F F F
62062024-02-04T14:03:00.964Z 22 440-459 53 53 53 82 82 82 T F T <---
62072024-02-04T14:03:00.964Z 23 460-479 54 54 54 87 87 87 T F T <---
62082024-02-04T14:03:00.964Z 24 480-499 56 47 56 89 75 89 T F T <---
62092024-02-04T14:03:00.964Z 25 500-519 56 48 56 89 76 89 T F T <---
62102024-02-04T14:03:00.964Z 26 520-539 50 50 50 77 77 77 T F T <---
62112024-02-04T14:03:00.964Z 27 540-559 50 50 50 77 77 77 F F F
62122024-02-04T14:03:00.964Z 28 560-579 52 52 52 81 81 81 F F F
62132024-02-04T14:03:00.964Z 29 580-599 52 52 52 81 81 81 F F F
62142024-02-04T14:03:00.964ZMax gen: 56, Max flush: 90
62152024-02-04T14:03:00.964ZError: Difference in extent metadata found!
62162024-02-04T14:03:00.968ZA Difference in extent metadata is expected here
62172024-02-04T14:03:00.968ZOn loop 28
62182024-02-04T14:03:00.968Z
62192024-02-04T14:03:00.971ZVerifying data now
62202024-02-04T14:03:06.049ZNo change after verify
62212024-02-04T14:03:06.049ZLoop: 28 Downstairs dump after verify (and repair):
62222024-02-04T14:03:06.072ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62232024-02-04T14:03:06.072Z 0 000-019 54 54 54 87 87 87 F F F
62242024-02-04T14:03:06.072Z 1 020-039 56 56 56 89 89 89 F F F
62252024-02-04T14:03:06.072Z 2 040-059 54 54 54 83 83 83 F F F
62262024-02-04T14:03:06.072Z 3 060-079 56 56 56 89 89 89 F F F
62272024-02-04T14:03:06.072Z 4 080-099 53 53 53 82 82 82 F F F
62282024-02-04T14:03:06.072Z 5 100-119 53 53 53 82 82 82 F F F
62292024-02-04T14:03:06.075Z 6 120-139 51 51 51 78 78 78 F F F
62302024-02-04T14:03:06.075Z 7 140-159 56 56 56 90 90 90 F F F
62312024-02-04T14:03:06.075Z 8 160-179 54 54 54 87 87 87 F F F
62322024-02-04T14:03:06.075Z 9 180-199 54 54 54 84 84 84 F F F
62332024-02-04T14:03:06.075Z 10 200-219 48 48 48 76 76 76 F F F
62342024-02-04T14:03:06.075Z 11 220-239 56 56 56 89 89 89 F F F
62352024-02-04T14:03:06.075Z 12 240-259 52 52 52 80 80 80 F F F
62362024-02-04T14:03:06.075Z 13 260-279 44 44 44 72 72 72 F F F
62372024-02-04T14:03:06.075Z 14 280-299 54 54 54 87 87 87 F F F
62382024-02-04T14:03:06.076Z 15 300-319 52 52 52 81 81 81 F F F
62392024-02-04T14:03:06.076Z 16 320-339 51 51 51 78 78 78 F F F
62402024-02-04T14:03:06.076Z 17 340-359 57 57 57 91 91 91 F F F
62412024-02-04T14:03:06.076Z 18 360-379 56 56 56 89 89 89 F F F
62422024-02-04T14:03:06.076Z 19 380-399 57 57 57 91 91 91 F F F
62432024-02-04T14:03:06.076Z 20 400-419 55 55 55 88 88 88 F F F
62442024-02-04T14:03:06.076Z 21 420-439 55 55 55 88 88 88 F F F
62452024-02-04T14:03:06.076Z 22 440-459 57 57 57 91 91 91 F F F
62462024-02-04T14:03:06.076Z 23 460-479 57 57 57 91 91 91 F F F
62472024-02-04T14:03:06.076Z 24 480-499 57 57 57 91 91 91 F F F
62482024-02-04T14:03:06.076Z 25 500-519 57 57 57 91 91 91 F F F
62492024-02-04T14:03:06.076Z 26 520-539 57 57 57 91 91 91 F F F
62502024-02-04T14:03:06.076Z 27 540-559 50 50 50 77 77 77 F F F
62512024-02-04T14:03:06.076Z 28 560-579 52 52 52 81 81 81 F F F
62522024-02-04T14:03:06.076Z 29 580-599 52 52 52 81 81 81 F F F
62532024-02-04T14:03:06.076ZMax gen: 57, Max flush: 91
62542024-02-04T14:03:06.079Z
62552024-02-04T14:03:06.079ZBegin loop 29
62562024-02-04T14:03:06.079ZDownstairs to restart: 1
62572024-02-04T14:03:06.107Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.140573603Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361}
62582024-02-04T14:03:06.107Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141011049Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361}
62592024-02-04T14:03:06.108Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141040332Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361}
62602024-02-04T14:03:06.111Z{"msg":"Crucible 4fe80b53-ac36-4370-8b02-6c34958f04ea has session id: 352dc593-c800-41ee-a5a9-403f29e0835d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141075264Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62612024-02-04T14:03:06.111Z{"msg":"Upstairs opts: Upstairs UUID: 4fe80b53-ac36-4370-8b02-6c34958f04ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141178051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62622024-02-04T14:03:06.111Z{"msg":"Crucible stats registered with UUID: 4fe80b53-ac36-4370-8b02-6c34958f04ea","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141248718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62632024-02-04T14:03:06.111ZCrucible runtime is spawned
62642024-02-04T14:03:06.111Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141505796Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361}
62652024-02-04T14:03:06.111Z{"msg":"4fe80b53-ac36-4370-8b02-6c34958f04ea active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141575949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62662024-02-04T14:03:06.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141606433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62672024-02-04T14:03:06.111Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141671701Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62682024-02-04T14:03:06.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141697583Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62692024-02-04T14:03:06.111Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141757225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62702024-02-04T14:03:06.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141790124Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62712024-02-04T14:03:06.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141828265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62722024-02-04T14:03:06.112Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141894443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62732024-02-04T14:03:06.112Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.141939353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62742024-02-04T14:03:06.112Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.14208257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62752024-02-04T14:03:06.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.142203433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62762024-02-04T14:03:06.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.142326178Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62772024-02-04T14:03:06.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.142376511Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62782024-02-04T14:03:06.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.143437522Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62792024-02-04T14:03:06.112Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.143507509Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62802024-02-04T14:03:06.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.143533995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62812024-02-04T14:03:06.112Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.143557116Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62822024-02-04T14:03:06.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.144206537Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62832024-02-04T14:03:06.112Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.144286901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62842024-02-04T14:03:06.116Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.144766129Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62852024-02-04T14:03:06.116Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.144939938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62862024-02-04T14:03:06.116Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.145453064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62872024-02-04T14:03:06.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.145719892Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62882024-02-04T14:03:06.116Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.145806424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62892024-02-04T14:03:06.116Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.145835444Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62902024-02-04T14:03:06.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.145891781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62912024-02-04T14:03:06.116Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.145924074Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62922024-02-04T14:03:06.116Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.145978678Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62932024-02-04T14:03:06.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146350925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62942024-02-04T14:03:06.116Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146418273Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62952024-02-04T14:03:06.116Z{"msg":"[0]R flush_numbers[0..12]: [87, 89, 83, 89, 82, 82, 78, 90, 87, 84, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146464847Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62962024-02-04T14:03:06.116Z{"msg":"[0]R generation[0..12]: [54, 56, 54, 56, 53, 53, 51, 56, 54, 54, 48, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146531965Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62972024-02-04T14:03:06.116Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.1466023Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62982024-02-04T14:03:06.116Z{"msg":"[1]R flush_numbers[0..12]: [87, 89, 83, 89, 82, 82, 78, 90, 87, 84, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146646274Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
62992024-02-04T14:03:06.116Z{"msg":"[1]R generation[0..12]: [54, 56, 54, 56, 53, 53, 51, 56, 54, 54, 48, 56]","v":Wait for a query_work_queue command to finish before sending IO
63002024-02-04T14:03:06.116Z0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146729254Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63012024-02-04T14:03:06.117Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.1467949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63022024-02-04T14:03:06.117Z{"msg":"[2]R flush_numbers[0..12]: [87, 89, 83, 89, 82, 82, 78, 90, 87, 84, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146840273Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63032024-02-04T14:03:06.117Z{"msg":"[2]R generation[0..12]: [54, 56, 54, 56, 53, 53, 51, 56, 54, 54, 48, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146900447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63042024-02-04T14:03:06.117Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.146945472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63052024-02-04T14:03:06.117Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.14700717Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63062024-02-04T14:03:06.117Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.147051365Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63072024-02-04T14:03:06.117Z{"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.14710366Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63082024-02-04T14:03:06.117Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.147144249Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63092024-02-04T14:03:06.117Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.147204045Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63102024-02-04T14:03:06.117Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.1472423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63112024-02-04T14:03:06.117Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.147336318Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63122024-02-04T14:03:06.117Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.147368826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63132024-02-04T14:03:06.117Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.147439019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63142024-02-04T14:03:06.117Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.14747565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63152024-02-04T14:03:06.117Z"pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63162024-02-04T14:03:06.117Z{"msg":"4fe80b53-ac36-4370-8b02-6c34958f04ea is now active with session: 352dc593-c800-41ee-a5a9-403f29e0835d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.147608715Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63172024-02-04T14:03:06.117Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.147635854Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361,"session_id":"352dc593-c800-41ee-a5a9-403f29e0835d"}
63182024-02-04T14:03:06.117Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:05.14767593Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361}
63192024-02-04T14:03:06.117ZLoading write count information from file "/tmp/test_repair_verify.data"
63202024-02-04T14:03:06.117ZRun Repair workload
63212024-02-04T14:03:06.117Z01/30 Write block 67 len 20480 data: 5 5 4 4 4
63222024-02-04T14:03:06.123Z02/30 Write block 309 len 28672 data: 5 5 6 5 6 5 3
63232024-02-04T14:03:06.123Z03/30 Read block 386 len 28672
63242024-02-04T14:03:06.126Z04/30 Write block 40 len 36864 data: 4 3 5 4 5 5 5 4 5
63252024-02-04T14:03:06.126Z05/30 Write block 149 len 36864 data: 5 5 5 5 5 4 3 3 3
63262024-02-04T14:03:06.126Z06/30 Read block 129 len 16384
63272024-02-04T14:03:06.129Z07/30 Read block 254 len 28672
63282024-02-04T14:03:06.129Z08/30 Read block 167 len 36864
63292024-02-04T14:03:06.132Z09/30 Write block 25 len 36864 data: 4 4 4 5 5 5 6 4 5
63302024-02-04T14:03:06.132Z10/30 Write block 145 len 8192 data: 5 5
63312024-02-04T14:03:06.135Z11/30 Write block 320 len 16384 data: 5 4 4 4
63322024-02-04T14:03:06.135Z12/30 Read block 135 len 4096
63332024-02-04T14:03:06.135Z13/30 Read block 362 len 4096
63342024-02-04T14:03:06.140Z14/30 Write block 419 len 32768 data: 6 9 9 8 9 5 6 6
63352024-02-04T14:03:06.140Z15/30 Read block 175 len 28672
63362024-02-04T14:03:06.144Z16/30 Read block 253 len 40960
63372024-02-04T14:03:06.144Z17/30 Flush
63382024-02-04T14:03:06.146Z18/30 Write block 555 len 24576 data: 4 6 7 6 6 7
63392024-02-04T14:03:06.147Z19/30 Write block 536 len 40960 data: 6 4 4 4 5 6 4 6 4 4
63402024-02-04T14:03:06.150Z20/30 Read block 298 len 40960
63412024-02-04T14:03:06.152Z21/30 Flush
63422024-02-04T14:03:06.155Z22/30 Write block 85 len 32768 data: 6 5 5 4 3 3 3 2
63432024-02-04T14:03:06.158Z23/30 Read block 413 len 16384
63442024-02-04T14:03:06.159Z24/30 Read block 368 len 28672
63452024-02-04T14:03:06.161Z25/30 Read block 200 len 36864
63462024-02-04T14:03:06.161Z26/30 Read block 260 len 32768
63472024-02-04T14:03:06.164Z27/30 Write block 380 len 40960 data: 9 7 4 5 5 4 4 5 6 6
63482024-02-04T14:03:06.167Z28/30 Write block 393 len 40960 data: 7 7 5 5 7 6 7 7 6 5
63492024-02-04T14:03:06.167Z29/30 Read block 287 len 8192
63502024-02-04T14:03:06.173Z30/30 Read block 563 len 28672
63512024-02-04T14:03:06.173Z----------------------------------------------------------------
63522024-02-04T14:03:06.173Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63532024-02-04T14:03:06.173ZStates: Active Active Active
63542024-02-04T14:03:06.173ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63552024-02-04T14:03:06.173Z 1 Acked 1000 Write 5 Done Sent Done false
63562024-02-04T14:03:06.173Z 2 Acked 1001 Write 7 Done Sent Done false
63572024-02-04T14:03:06.173Z 3 Acked 1002 Read 7 Done Sent Done false
63582024-02-04T14:03:06.173Z 4 Acked 1003 Write 9 Done Sent Done false
63592024-02-04T14:03:06.173Z 5 Acked 1004 Write 9 Done Sent Done false
63602024-02-04T14:03:06.173Z 6 Acked 1005 Read 4 Done Sent Done false
63612024-02-04T14:03:06.173Z 7 Acked 1006 Read 7 Done Sent Done false
63622024-02-04T14:03:06.173Z 8 Acked 1007 Read 9 Done Sent Done false
63632024-02-04T14:03:06.173Z 9 Acked 1008 Write 9 Done Sent Done false
63642024-02-04T14:03:06.173Z 10 Acked 1009 Write 2 Done Sent Done false
63652024-02-04T14:03:06.174Z 11 Acked 1010 Write 4 Done Sent Done false
63662024-02-04T14:03:06.174Z 12 Acked 1011 Read 1 Done Sent Done false
63672024-02-04T14:03:06.174Z 13 Acked 1012 Read 1 Done Sent Done false
63682024-02-04T14:03:06.174Z 14 Acked 1013 Write 8 Done Sent Done false
63692024-02-04T14:03:06.174Z 15 Acked 1014 Read 7 Done Sent Done false
63702024-02-04T14:03:06.174Z 16 Acked 1015 Read 10 Done Sent Done false
63712024-02-04T14:03:06.174Z 17 Acked 1016 Flush 0 Done Sent Done false
63722024-02-04T14:03:06.174Z 18 Acked 1017 Write 6 Done Sent Done false
63732024-02-04T14:03:06.174Z 19 Acked 1018 Write 10 Done Sent Done false
63742024-02-04T14:03:06.174Z 20 Acked 1019 Read 10 Done Sent Done false
63752024-02-04T14:03:06.174Z 21 Acked 1020 Flush 0 Done Sent Done false
63762024-02-04T14:03:06.174Z 22 Acked 1021 Write 8 Done Sent Done false
63772024-02-04T14:03:06.174Z 23 Acked 1022 Read 4 Done Sent Done false
63782024-02-04T14:03:06.174Z 24 Acked 1023 Read 7 Done Sent Done false
63792024-02-04T14:03:06.174Z 25 Acked 1024 Read 9 Done Sent Done false
63802024-02-04T14:03:06.174Z 26 Acked 1025 Read 8 Done Sent Done false
63812024-02-04T14:03:06.174Z 27 Acked 1026 Write 10 Done Sent Done false
63822024-02-04T14:03:06.174Z 28 Acked 1027 Write 10 Done Sent Done false
63832024-02-04T14:03:06.174Z 29 Acked 1028 Read 2 Done Sent Done false
63842024-02-04T14:03:06.174Z 30 Acked 1029 Read 7 Done Sent Done false
63852024-02-04T14:03:06.174Z STATES DS:0 DS:1 DS:2 TOTAL
63862024-02-04T14:03:06.174Z New 0 0 0 0
63872024-02-04T14:03:06.174Z Sent 0 30 0 30
63882024-02-04T14:03:06.174Z Done 30 0 30 60
63892024-02-04T14:03:06.174Z Skipped 0 0 0 0
63902024-02-04T14:03:06.175Z Error 0 0 0 0
63912024-02-04T14:03:06.175ZLast Flush: 1020 0 1020
63922024-02-04T14:03:06.175ZDownstairs last five completed:
63932024-02-04T14:03:06.175ZUpstairs last five completed: 30 29 28 27 26
63942024-02-04T14:03:06.178Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:05.20379752Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1361}
63952024-02-04T14:03:06.178ZWrote out file "/tmp/test_repair_verify.data"
63962024-02-04T14:03:06.178Z
63972024-02-04T14:03:06.183ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63982024-02-04T14:03:06.208ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63992024-02-04T14:03:06.208Z 0 000-019 54 54 54 87 87 87 F F F
64002024-02-04T14:03:06.208Z 1 020-039 58 56 58 92 89 92 F F F <---
64012024-02-04T14:03:06.208Z 2 040-059 58 54 58 92 83 92 F F F <---
64022024-02-04T14:03:06.209Z 3 060-079 58 56 58 92 89 92 F F F <---
64032024-02-04T14:03:06.209Z 4 080-099 53 53 53 82 82 82 T F T <---
64042024-02-04T14:03:06.209Z 5 100-119 53 53 53 82 82 82 F F F
64052024-02-04T14:03:06.209Z 6 120-139 51 51 51 78 78 78 F F F
64062024-02-04T14:03:06.209Z 7 140-159 58 56 58 92 90 92 F F F <---
64072024-02-04T14:03:06.209Z 8 160-179 54 54 54 87 87 87 F F F
64082024-02-04T14:03:06.209Z 9 180-199 54 54 54 84 84 84 F F F
64092024-02-04T14:03:06.209Z 10 200-219 48 48 48 76 76 76 F F F
64102024-02-04T14:03:06.212Z 11 220-239 56 56 56 89 89 89 F F F
64112024-02-04T14:03:06.212Z 12 240-259 52 52 52 80 80 80 F F F
64122024-02-04T14:03:06.212Z 13 260-279 44 44 44 72 72 72 F F F
64132024-02-04T14:03:06.212Z 14 280-299 54 54 54 87 87 87 F F F
64142024-02-04T14:03:06.212Z 15 300-319 58 52 58 92 81 92 F F F <---
64152024-02-04T14:03:06.212Z 16 320-339 58 51 58 92 78 92 F F F <---
64162024-02-04T14:03:06.212Z 17 340-359 57 57 57 91 91 91 F F F
64172024-02-04T14:03:06.212Z 18 360-379 56 56 56 89 89 89 F F F
64182024-02-04T14:03:06.212Z 19 380-399 57 57 57 91 91 91 T F T <---
64192024-02-04T14:03:06.212Z 20 400-419 58 55 58 92 88 92 T F T <---
64202024-02-04T14:03:06.212Z 21 420-439 58 55 58 92 88 92 F F F <---
64212024-02-04T14:03:06.212Z 22 440-459 57 57 57 91 91 91 F F F
64222024-02-04T14:03:06.212Z 23 460-479 57 57 57 91 91 91 F F F
64232024-02-04T14:03:06.213Z 24 480-499 57 57 57 91 91 91 F F F
64242024-02-04T14:03:06.213Z 25 500-519 57 57 57 91 91 91 F F F
64252024-02-04T14:03:06.213Z 26 520-539 58 57 58 93 91 93 F F F <---
64262024-02-04T14:03:06.213Z 27 540-559 58 50 58 93 77 93 F F F <---
64272024-02-04T14:03:06.213Z 28 560-579 58 52 58 93 81 93 F F F <---
64282024-02-04T14:03:06.213Z 29 580-599 52 52 52 81 81 81 F F F
64292024-02-04T14:03:06.213ZMax gen: 58, Max flush: 93
64302024-02-04T14:03:06.213ZError: Difference in extent metadata found!
64312024-02-04T14:03:06.216ZA Difference in extent metadata is expected here
64322024-02-04T14:03:06.216ZOn loop 29
64332024-02-04T14:03:06.216Z
64342024-02-04T14:03:06.218ZVerifying data now
64352024-02-04T14:03:11.354ZNo change after verify
64362024-02-04T14:03:11.354ZLoop: 29 Downstairs dump after verify (and repair):
64372024-02-04T14:03:11.376ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64382024-02-04T14:03:11.376Z 0 000-019 54 54 54 87 87 87 F F F
64392024-02-04T14:03:11.376Z 1 020-039 58 58 58 92 92 92 F F F
64402024-02-04T14:03:11.376Z 2 040-059 58 58 58 92 92 92 F F F
64412024-02-04T14:03:11.377Z 3 060-079 58 58 58 92 92 92 F F F
64422024-02-04T14:03:11.377Z 4 080-099 59 59 59 94 94 94 F F F
64432024-02-04T14:03:11.380Z 5 100-119 53 53 53 82 82 82 F F F
64442024-02-04T14:03:11.380Z 6 120-139 51 51 51 78 78 78 F F F
64452024-02-04T14:03:11.380Z 7 140-159 58 58 58 92 92 92 F F F
64462024-02-04T14:03:11.380Z 8 160-179 54 54 54 87 87 87 F F F
64472024-02-04T14:03:11.380Z 9 180-199 54 54 54 84 84 84 F F F
64482024-02-04T14:03:11.380Z 10 200-219 48 48 48 76 76 76 F F F
64492024-02-04T14:03:11.380Z 11 220-239 56 56 56 89 89 89 F F F
64502024-02-04T14:03:11.380Z 12 240-259 52 52 52 80 80 80 F F F
64512024-02-04T14:03:11.380Z 13 260-279 44 44 44 72 72 72 F F F
64522024-02-04T14:03:11.380Z 14 280-299 54 54 54 87 87 87 F F F
64532024-02-04T14:03:11.380Z 15 300-319 58 58 58 92 92 92 F F F
64542024-02-04T14:03:11.380Z 16 320-339 58 58 58 92 92 92 F F F
64552024-02-04T14:03:11.380Z 17 340-359 57 57 57 91 91 91 F F F
64562024-02-04T14:03:11.380Z 18 360-379 56 56 56 89 89 89 F F F
64572024-02-04T14:03:11.380Z 19 380-399 59 59 59 94 94 94 F F F
64582024-02-04T14:03:11.380Z 20 400-419 59 59 59 94 94 94 F F F
64592024-02-04T14:03:11.380Z 21 420-439 58 58 58 92 92 92 F F F
64602024-02-04T14:03:11.380Z 22 440-459 57 57 57 91 91 91 F F F
64612024-02-04T14:03:11.381Z 23 460-479 57 57 57 91 91 91 F F F
64622024-02-04T14:03:11.381Z 24 480-499 57 57 57 91 91 91 F F F
64632024-02-04T14:03:11.381Z 25 500-519 57 57 57 91 91 91 F F F
64642024-02-04T14:03:11.381Z 26 520-539 58 58 58 93 93 93 F F F
64652024-02-04T14:03:11.381Z 27 540-559 58 58 58 93 93 93 F F F
64662024-02-04T14:03:11.381Z 28 560-579 58 58 58 93 93 93 F F F
64672024-02-04T14:03:11.381Z 29 580-599 52 52 52 81 81 81 F F F
64682024-02-04T14:03:11.381ZMax gen: 59, Max flush: 94
64692024-02-04T14:03:11.384Z
64702024-02-04T14:03:11.384ZBegin loop 30
64712024-02-04T14:03:11.384ZDownstairs to restart: 1
64722024-02-04T14:03:11.411Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.443839548Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369}
64732024-02-04T14:03:11.411Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.444350877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369}
64742024-02-04T14:03:11.414Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.444386564Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369}
64752024-02-04T14:03:11.414Z{"msg":"Crucible f626feb8-6952-4cd0-92b2-ef25beeeda44 has session id: 2749ab9b-c49a-4d91-a74c-7441a9f1559c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.444499027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64762024-02-04T14:03:11.414Z{"msg":"Crucible runtime is spawned
64772024-02-04T14:03:11.414ZUpstairs opts: Upstairs UUID: f626feb8-6952-4cd0-92b2-ef25beeeda44, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.444765202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64782024-02-04T14:03:11.414Z{"msg":"Crucible stats registered with UUID: f626feb8-6952-4cd0-92b2-ef25beeeda44","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.444851756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64792024-02-04T14:03:11.414Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.444973434Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369}
64802024-02-04T14:03:11.414Z{"msg":"f626feb8-6952-4cd0-92b2-ef25beeeda44 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445059939Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64812024-02-04T14:03:11.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.44510727Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64822024-02-04T14:03:11.415Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445162401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64832024-02-04T14:03:11.415Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445188337Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64842024-02-04T14:03:11.415Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445212111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64852024-02-04T14:03:11.415Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.44523491Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64862024-02-04T14:03:11.415Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445267084Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64872024-02-04T14:03:11.415Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445339539Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"io task","client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64882024-02-04T14:03:11.415Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445388336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"io task","client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64892024-02-04T14:03:11.415Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445459022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"io task","client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64902024-02-04T14:03:11.415Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445657413Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"io task","client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64912024-02-04T14:03:11.415Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445750044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"io task","client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64922024-02-04T14:03:11.415Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.445794291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"io task","client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64932024-02-04T14:03:11.415Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.446992068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64942024-02-04T14:03:11.415Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.447067624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64952024-02-04T14:03:11.415Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.447114032Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64962024-02-04T14:03:11.415Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.447174687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64972024-02-04T14:03:11.415Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.447225572Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64982024-02-04T14:03:11.415Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.447338624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
64992024-02-04T14:03:11.419Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.448016662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65002024-02-04T14:03:11.419Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.448111983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65012024-02-04T14:03:11.419Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.44839438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65022024-02-04T14:03:11.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.448986747Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65032024-02-04T14:03:11.419Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449111839Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65042024-02-04T14:03:11.420Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449193134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65052024-02-04T14:03:11.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449232149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65062024-02-04T14:03:11.420Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449275847Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65072024-02-04T14:03:11.420Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449338597Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65082024-02-04T14:03:11.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449380577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65092024-02-04T14:03:11.420Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449436236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65102024-02-04T14:03:11.420Z{"msg":"[0]R flush_numbers[0..12]: [87, 92, 92, 92, 94, 82, 78, 92, 87, 84, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449481755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65112024-02-04T14:03:11.420Z{"msg":"[0]R generation[0..12]: [54, 58, 58, 58, 59, 53, 51, 58, 54, 54, 48, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449536798Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65122024-02-04T14:03:11.420Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.44958243Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"Wait for a query_work_queue command to finish before sending IO
65132024-02-04T14:03:11.420Z}
65142024-02-04T14:03:11.420Z{"msg":"[1]R flush_numbers[0..12]: [87, 92, 92, 92, 94, 82, 78, 92, 87, 84, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449793702Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65152024-02-04T14:03:11.420Z{"msg":"[1]R generation[0..12]: [54, 58, 58, 58, 59, 53, 51, 58, 54, 54, 48, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449869693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65162024-02-04T14:03:11.420Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449915442Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65172024-02-04T14:03:11.420Z{"msg":"[2]R flush_numbers[0..12]: [87, 92, 92, 92, 94, 82, 78, 92, 87, 84, 76, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.449973567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65182024-02-04T14:03:11.420Z{"msg":"[2]R generation[0..12]: [54, 58, 58, 58, 59, 53, 51, 58, 54, 54, 48, 56]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450019281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65192024-02-04T14:03:11.420Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450083781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65202024-02-04T14:03:11.420Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450127103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65212024-02-04T14:03:11.420Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450190683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65222024-02-04T14:03:11.420Z{"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450232773Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65232024-02-04T14:03:11.420Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450293231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65242024-02-04T14:03:11.420Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450336138Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65252024-02-04T14:03:11.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65262024-02-04T14:03:11.420Z,"time":"2024-02-04T14:03:10.450394897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65272024-02-04T14:03:11.420Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450472458Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65282024-02-04T14:03:11.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450514741Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65292024-02-04T14:03:11.421Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450557498Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65302024-02-04T14:03:11.421Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.45061689Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65312024-02-04T14:03:11.421Z{"msg":"f626feb8-6952-4cd0-92b2-ef25beeeda44 is now active with session: 2749ab9b-c49a-4d91-a74c-7441a9f1559c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450660021Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65322024-02-04T14:03:11.421Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450711699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369,"session_id":"2749ab9b-c49a-4d91-a74c-7441a9f1559c"}
65332024-02-04T14:03:11.421Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:10.450751973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369}
65342024-02-04T14:03:11.421ZLoading write count information from file "/tmp/test_repair_verify.data"
65352024-02-04T14:03:11.421ZRun Repair workload
65362024-02-04T14:03:11.421Z01/30 Write block 590 len 24576 data: 2 2 3 3 3 3
65372024-02-04T14:03:11.424Z02/30 Write block 516 len 16384 data: 8 6 6 6
65382024-02-04T14:03:11.424Z03/30 Write block 431 len 36864 data: 5 5 5 5 5 4 4 4 3
65392024-02-04T14:03:11.424Z04/30 Write block 73 len 40960 data: 4 5 5 5 6 5 6 5 6 6
65402024-02-04T14:03:11.430Z05/30 Read block 51 len 20480
65412024-02-04T14:03:11.433Z06/30 Read block 167 len 24576
65422024-02-04T14:03:11.433Z07/30 Read block 282 len 36864
65432024-02-04T14:03:11.433Z08/30 Write block 60 len 20480 data: 5 5 5 4 5
65442024-02-04T14:03:11.433Z09/30 Write block 517 len 4096 data: 7
65452024-02-04T14:03:11.435Z10/30 Read block 242 len 36864
65462024-02-04T14:03:11.436Z11/30 Write block 284 len 28672 data: 9 8 6 7 8 7 9
65472024-02-04T14:03:11.438Z12/30 Write block 237 len 8192 data: 5 5
65482024-02-04T14:03:11.438Z13/30 Write block 107 len 36864 data: 6 10 9 9 10 12 10 10 10
65492024-02-04T14:03:11.438Z14/30 Read block 7 len 36864
65502024-02-04T14:03:11.444Z15/30 Write block 262 len 4096 data: 6
65512024-02-04T14:03:11.444Z16/30 Flush
65522024-02-04T14:03:11.447Z17/30 Write block 199 len 40960 data: 7 7 7 7 7 6 5 5 4 4
65532024-02-04T14:03:11.449Z18/30 Write block 446 len 8192 data: 3 3
65542024-02-04T14:03:11.449Z19/30 Read block 494 len 12288
65552024-02-04T14:03:11.452Z20/30 Read block 363 len 20480
65562024-02-04T14:03:11.452Z21/30 Write block 250 len 20480 data: 10 9 9 6 6
65572024-02-04T14:03:11.452Z22/30 Write block 91 len 8192 data: 4 3
65582024-02-04T14:03:11.455Z23/30 Write block 463 len 24576 data: 6 5 7 9 8 6
65592024-02-04T14:03:11.455Z24/30 Write block 182 len 16384 data: 7 9 7 8
65602024-02-04T14:03:11.455Z25/30 Read block 357 len 4096
65612024-02-04T14:03:11.458Z26/30 Flush
65622024-02-04T14:03:11.461Z27/30 Write block 51 len 40960 data: 4 4 4 3 3 2 3 3 4 6
65632024-02-04T14:03:11.464Z28/30 Write block 341 len 28672 data: 4 3 3 3 3 4 4
65642024-02-04T14:03:11.467Z29/30 Write block 409 len 36864 data: 5 5 7 6 8 8 8 6 6
65652024-02-04T14:03:11.467Z30/30 Read block 44 len 36864
65662024-02-04T14:03:11.470Z----------------------------------------------------------------
65672024-02-04T14:03:11.470Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65682024-02-04T14:03:11.470ZStates: Active Active Active
65692024-02-04T14:03:11.470ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65702024-02-04T14:03:11.470Z 1 Acked 1000 Write 6 Done Done Done false
65712024-02-04T14:03:11.470Z 2 Acked 1001 Write 4 Done Done Done false
65722024-02-04T14:03:11.471Z 3 Acked 1002 Write 9 Done Sent Done false
65732024-02-04T14:03:11.471Z 4 Acked 1003 Write 10 Done Sent Done false
65742024-02-04T14:03:11.471Z 5 Acked 1004 Read 5 Done Sent Done false
65752024-02-04T14:03:11.471Z 6 Acked 1005 Read 6 Done Sent Done false
65762024-02-04T14:03:11.471Z 7 Acked 1006 Read 9 Done Sent Done false
65772024-02-04T14:03:11.471Z 8 Acked 1007 Write 5 Done Sent Done false
65782024-02-04T14:03:11.471Z 9 Acked 1008 Write 1 Done Sent Done false
65792024-02-04T14:03:11.471Z 10 Acked 1009 Read 9 Done Sent Done false
65802024-02-04T14:03:11.471Z 11 Acked 1010 Write 7 Done Sent Done false
65812024-02-04T14:03:11.471Z 12 Acked 1011 Write 2 Done Sent Done false
65822024-02-04T14:03:11.471Z 13 Acked 1012 Write 9 Done Sent Done false
65832024-02-04T14:03:11.471Z 14 Acked 1013 Read 9 Done Sent Done false
65842024-02-04T14:03:11.471Z 15 Acked 1014 Write 1 Done Sent Done false
65852024-02-04T14:03:11.471Z 16 Acked 1015 Flush 0 Done Sent Done false
65862024-02-04T14:03:11.471Z 17 Acked 1016 Write 10 Done Sent Done false
65872024-02-04T14:03:11.471Z 18 Acked 1017 Write 2 Done Sent Done false
65882024-02-04T14:03:11.471Z 19 Acked 1018 Read 3 Done Sent Done false
65892024-02-04T14:03:11.471Z 20 Acked 1019 Read 5 Done Sent Done false
65902024-02-04T14:03:11.471Z 21 Acked 1020 Write 5 Done Sent Done false
65912024-02-04T14:03:11.471Z 22 Acked 1021 Write 2 Done Sent Done false
65922024-02-04T14:03:11.471Z 23 Acked 1022 Write 6 Done Sent Done false
65932024-02-04T14:03:11.471Z 24 Acked 1023 Write 4 Done Sent Done false
65942024-02-04T14:03:11.471Z 25 Acked 1024 Read 1 Done Sent Done false
65952024-02-04T14:03:11.471Z 26 Acked 1025 Flush 0 Done Sent Done false
65962024-02-04T14:03:11.472Z 27 Acked 1026 Write 10 Done Sent Done false
65972024-02-04T14:03:11.472Z 28 Acked 1027 Write 7 Done Sent Done false
65982024-02-04T14:03:11.472Z 29 Acked 1028 Write 9 Done Sent Done false
65992024-02-04T14:03:11.472Z 30 Acked 1029 Read 9 Done Sent Done false
66002024-02-04T14:03:11.472Z STATES DS:0 DS:1 DS:2 TOTAL
66012024-02-04T14:03:11.472Z New 0 0 0 0
66022024-02-04T14:03:11.472Z Sent 0 28 0 28
66032024-02-04T14:03:11.472Z Done 30 2 30 62
66042024-02-04T14:03:11.472Z Skipped 0 0 0 0
66052024-02-04T14:03:11.472Z Error 0 0 0 0
66062024-02-04T14:03:11.472ZLast Flush: 1025 0 1025
66072024-02-04T14:03:11.472ZDownstairs last five completed:
66082024-02-04T14:03:11.472ZUpstairs last five completed: 30 29 28 27 26
66092024-02-04T14:03:11.472Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:10.501306527Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1369}
66102024-02-04T14:03:11.472ZWrote out file "/tmp/test_repair_verify.data"
66112024-02-04T14:03:11.475Z
66122024-02-04T14:03:11.479ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66132024-02-04T14:03:11.504ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66142024-02-04T14:03:11.504Z 0 000-019 54 54 54 87 87 87 F F F
66152024-02-04T14:03:11.504Z 1 020-039 58 58 58 92 92 92 F F F
66162024-02-04T14:03:11.504Z 2 040-059 58 58 58 92 92 92 T F T <---
66172024-02-04T14:03:11.504Z 3 060-079 60 58 60 95 92 95 T F T <---
66182024-02-04T14:03:11.504Z 4 080-099 60 59 60 96 94 96 F F F <---
66192024-02-04T14:03:11.504Z 5 100-119 60 53 60 95 82 95 F F F <---
66202024-02-04T14:03:11.504Z 6 120-139 51 51 51 78 78 78 F F F
66212024-02-04T14:03:11.504Z 7 140-159 58 58 58 92 92 92 F F F
66222024-02-04T14:03:11.504Z 8 160-179 54 54 54 87 87 87 F F F
66232024-02-04T14:03:11.504Z 9 180-199 60 54 60 96 84 96 F F F <---
66242024-02-04T14:03:11.507Z 10 200-219 60 48 60 96 76 96 F F F <---
66252024-02-04T14:03:11.507Z 11 220-239 60 56 60 95 89 95 F F F <---
66262024-02-04T14:03:11.507Z 12 240-259 60 52 60 96 80 96 F F F <---
66272024-02-04T14:03:11.508Z 13 260-279 60 44 60 95 72 95 F F F <---
66282024-02-04T14:03:11.508Z 14 280-299 60 54 60 95 87 95 F F F <---
66292024-02-04T14:03:11.508Z 15 300-319 58 58 58 92 92 92 F F F
66302024-02-04T14:03:11.508Z 16 320-339 58 58 58 92 92 92 F F F
66312024-02-04T14:03:11.508Z 17 340-359 57 57 57 91 91 91 T F T <---
66322024-02-04T14:03:11.508Z 18 360-379 56 56 56 89 89 89 F F F
66332024-02-04T14:03:11.508Z 19 380-399 59 59 59 94 94 94 F F F
66342024-02-04T14:03:11.508Z 20 400-419 59 59 59 94 94 94 T F T <---
66352024-02-04T14:03:11.508Z 21 420-439 60 58 60 95 92 95 F F F <---
66362024-02-04T14:03:11.508Z 22 440-459 60 57 60 96 91 96 F F F <---
66372024-02-04T14:03:11.508Z 23 460-479 60 57 60 96 91 96 F F F <---
66382024-02-04T14:03:11.508Z 24 480-499 57 57 57 91 91 91 F F F
66392024-02-04T14:03:11.508Z 25 500-519 60 57 60 95 91 95 F T F <---
66402024-02-04T14:03:11.508Z 26 520-539 58 58 58 93 93 93 F F F
66412024-02-04T14:03:11.508Z 27 540-559 58 58 58 93 93 93 F F F
66422024-02-04T14:03:11.508Z 28 560-579 58 58 58 93 93 93 F F F
66432024-02-04T14:03:11.508Z 29 580-599 60 52 60 95 81 95 F T F <---
66442024-02-04T14:03:11.508ZMax gen: 60, Max flush: 96
66452024-02-04T14:03:11.508ZError: Difference in extent metadata found!
66462024-02-04T14:03:11.511ZA Difference in extent metadata is expected here
66472024-02-04T14:03:11.511ZOn loop 30
66482024-02-04T14:03:11.511Z
66492024-02-04T14:03:11.514ZVerifying data now
66502024-02-04T14:03:17.932ZNo change after verify
66512024-02-04T14:03:17.932ZLoop: 30 Downstairs dump after verify (and repair):
66522024-02-04T14:03:17.955ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66532024-02-04T14:03:17.955Z 0 000-019 54 54 54 87 87 87 F F F
66542024-02-04T14:03:17.955Z 1 020-039 58 58 58 92 92 92 F F F
66552024-02-04T14:03:17.955Z 2 040-059 61 61 61 97 97 97 F F F
66562024-02-04T14:03:17.955Z 3 060-079 61 61 61 97 97 97 F F F
66572024-02-04T14:03:17.955Z 4 080-099 60 60 60 96 96 96 F F F
66582024-02-04T14:03:17.955Z 5 100-119 60 60 60 95 95 95 F F F
66592024-02-04T14:03:17.955Z 6 120-139 51 51 51 78 78 78 F F F
66602024-02-04T14:03:17.955Z 7 140-159 58 58 58 92 92 92 F F F
66612024-02-04T14:03:17.958Z 8 160-179 54 54 54 87 87 87 F F F
66622024-02-04T14:03:17.958Z 9 180-199 60 60 60 96 96 96 F F F
66632024-02-04T14:03:17.958Z 10 200-219 60 60 60 96 96 96 F F F
66642024-02-04T14:03:17.958Z 11 220-239 60 60 60 95 95 95 F F F
66652024-02-04T14:03:17.958Z 12 240-259 60 60 60 96 96 96 F F F
66662024-02-04T14:03:17.958Z 13 260-279 60 60 60 95 95 95 F F F
66672024-02-04T14:03:17.959Z 14 280-299 60 60 60 95 95 95 F F F
66682024-02-04T14:03:17.959Z 15 300-319 58 58 58 92 92 92 F F F
66692024-02-04T14:03:17.959Z 16 320-339 58 58 58 92 92 92 F F F
66702024-02-04T14:03:17.959Z 17 340-359 61 61 61 97 97 97 F F F
66712024-02-04T14:03:17.959Z 18 360-379 56 56 56 89 89 89 F F F
66722024-02-04T14:03:17.959Z 19 380-399 59 59 59 94 94 94 F F F
66732024-02-04T14:03:17.959Z 20 400-419 61 61 61 97 97 97 F F F
66742024-02-04T14:03:17.959Z 21 420-439 60 60 60 95 95 95 F F F
66752024-02-04T14:03:17.959Z 22 440-459 60 60 60 96 96 96 F F F
66762024-02-04T14:03:17.959Z 23 460-479 60 60 60 96 96 96 F F F
66772024-02-04T14:03:17.959Z 24 480-499 57 57 57 91 91 91 F F F
66782024-02-04T14:03:17.959Z 25 500-519 60 60 60 95 95 95 F F F
66792024-02-04T14:03:17.959Z 26 520-539 58 58 58 93 93 93 F F F
66802024-02-04T14:03:17.959Z 27 540-559 58 58 58 93 93 93 F F F
66812024-02-04T14:03:17.959Z 28 560-579 58 58 58 93 93 93 F F F
66822024-02-04T14:03:17.959Z 29 580-599 60 60 60 95 95 95 F F F
66832024-02-04T14:03:17.959ZMax gen: 61, Max flush: 97
66842024-02-04T14:03:17.962Z
66852024-02-04T14:03:17.962ZBegin loop 31
66862024-02-04T14:03:17.962ZDownstairs to restart: 1
66872024-02-04T14:03:17.990Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.02291059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377}
66882024-02-04T14:03:17.990Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023420365Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377}
66892024-02-04T14:03:17.990Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023497477Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377}
66902024-02-04T14:03:17.993Z{"msg":"Crucible a9b29d47-6bd0-4d8a-8b69-a011567d3534 has session id: 296dff85-1591-4a6a-ab4d-9d8e77c016e6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023536852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
66912024-02-04T14:03:17.994Z{"msg":"Crucible runtime is spawned
66922024-02-04T14:03:17.994ZUpstairs opts: Upstairs UUID: a9b29d47-6bd0-4d8a-8b69-a011567d3534, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023671972Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
66932024-02-04T14:03:17.994Z{"msg":"Crucible stats registered with UUID: a9b29d47-6bd0-4d8a-8b69-a011567d3534","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023740201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
66942024-02-04T14:03:17.994Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023765645Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377}
66952024-02-04T14:03:17.994Z{"msg":"a9b29d47-6bd0-4d8a-8b69-a011567d3534 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023820115Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
66962024-02-04T14:03:17.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023846325Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
66972024-02-04T14:03:17.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023932647Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
66982024-02-04T14:03:17.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.023976395Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
66992024-02-04T14:03:17.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.024072783Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67002024-02-04T14:03:17.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.024188557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67012024-02-04T14:03:17.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.024242918Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67022024-02-04T14:03:17.994Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.02444633Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"io task","client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67032024-02-04T14:03:17.994Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.024537515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"io task","client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67042024-02-04T14:03:17.994Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.024584034Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"io task","client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67052024-02-04T14:03:17.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.024682368Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"io task","client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67062024-02-04T14:03:17.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.02473072Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"io task","client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67072024-02-04T14:03:17.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.024805714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"io task","client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67082024-02-04T14:03:17.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.025431835Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67092024-02-04T14:03:17.994Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.025524677Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67102024-02-04T14:03:17.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.025604695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67112024-02-04T14:03:17.994Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.025651033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67122024-02-04T14:03:17.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.026354152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67132024-02-04T14:03:17.995Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.026416781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67142024-02-04T14:03:17.995Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.026443839Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67152024-02-04T14:03:17.995Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.026467774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67162024-02-04T14:03:17.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.027455498Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67172024-02-04T14:03:17.999Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.027527222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67182024-02-04T14:03:17.999Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.02757065Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67192024-02-04T14:03:17.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.02759571Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67202024-02-04T14:03:17.999Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.027618217Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67212024-02-04T14:03:17.999Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.027677823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67222024-02-04T14:03:17.999Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.027707012Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67232024-02-04T14:03:17.999Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028285134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67242024-02-04T14:03:17.999Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028351374Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67252024-02-04T14:03:17.999Z{"msg":"[0]R flush_numbers[0..12]: [87, 92, 97, 97, 96, 95, 78, 92, 87, 96, 96, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028379452Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67262024-02-04T14:03:17.999Z{"msg":"[0]R generation[0..12]: [54, 58, 61, 61, 60, 60, 51, 58, 54, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028422801Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67272024-02-04T14:03:17.999Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028449164Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67282024-02-04T14:03:17.999Z{"msg":"[1]R flush_numbers[0..12]: [87, 92, 97, 97, 96, 95, 78, 92, 87, 96, 96, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028504937Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67292024-02-04T14:03:17.999Z{"msg":"[1]R generation[0..12]: [54, 58, 61, 61, 60, 60, 51, 58, 54, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028529906Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67302024-02-04T14:03:17.999Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]",Wait for a query_work_queue command to finish before sending IO
67312024-02-04T14:03:17.999Z"v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028558408Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67322024-02-04T14:03:17.999Z{"msg":"[2]R flush_numbers[0..12]: [87, 92, 97, 97, 96, 95, 78, 92, 87, 96, 96, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028601099Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67332024-02-04T14:03:17.999Z{"msg":"[2]R generation[0..12]: [54, 58, 61, 61, 60, 60, 51, 58, 54, 60, 60, 60]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028626035Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67342024-02-04T14:03:17.999Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028684342Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67352024-02-04T14:03:17.999Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028751144Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67362024-02-04T14:03:17.999Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028803882Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67372024-02-04T14:03:18.000Z{"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028872472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67382024-02-04T14:03:18.000Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028901401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67392024-02-04T14:03:18.000Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028965879Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67402024-02-04T14:03:18.000Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.028993695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67412024-02-04T14:03:18.000Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.029063314Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67422024-02-04T14:03:18.000Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.029091625Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67432024-02-04T14:03:18.000Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67442024-02-04T14:03:18.000Z,"time":"2024-02-04T14:03:17.029160773Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67452024-02-04T14:03:18.000Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.029197663Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67462024-02-04T14:03:18.000Z{"msg":"a9b29d47-6bd0-4d8a-8b69-a011567d3534 is now active with session: 296dff85-1591-4a6a-ab4d-9d8e77c016e6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.029257238Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67472024-02-04T14:03:18.000Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.029282257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377,"session_id":"296dff85-1591-4a6a-ab4d-9d8e77c016e6"}
67482024-02-04T14:03:18.000Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:17.029322336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377}
67492024-02-04T14:03:18.000ZLoading write count information from file "/tmp/test_repair_verify.data"
67502024-02-04T14:03:18.000ZRun Repair workload
67512024-02-04T14:03:18.001Z01/30 Write block 234 len 4096 data: 5
67522024-02-04T14:03:18.004Z02/30 Write block 59 len 16384 data: 5 7 6 6
67532024-02-04T14:03:18.004Z03/30 Write block 497 len 20480 data: 6 8 8 8 8
67542024-02-04T14:03:18.004Z04/30 Write block 513 len 28672 data: 3 5 7 9 8 7 7
67552024-02-04T14:03:18.004Z05/30 Read block 180 len 36864
67562024-02-04T14:03:18.007Z06/30 Write block 407 len 8192 data: 4 4
67572024-02-04T14:03:18.007Z07/30 Read block 445 len 28672
67582024-02-04T14:03:18.007Z08/30 Read block 473 len 12288
67592024-02-04T14:03:18.011Z09/30 Write block 459 len 4096 data: 6
67602024-02-04T14:03:18.011Z10/30 Read block 335 len 28672
67612024-02-04T14:03:18.016Z11/30 Write block 179 len 24576 data: 8 8 8 8 10 8
67622024-02-04T14:03:18.016Z12/30 Write block 456 len 16384 data: 7 8 7 7
67632024-02-04T14:03:18.019Z13/30 Write block 197 len 36864 data: 5 7 8 8 8 8 8 7 6
67642024-02-04T14:03:18.019Z14/30 Write block 261 len 24576 data: 5 7 7 6 5 6
67652024-02-04T14:03:18.019Z15/30 Write block 514 len 8192 data: 6 8
67662024-02-04T14:03:18.019Z16/30 Read block 407 len 36864
67672024-02-04T14:03:18.019Z17/30 Read block 18 len 40960
67682024-02-04T14:03:18.022Z18/30 Read block 6 len 20480
67692024-02-04T14:03:18.022Z19/30 Read block 152 len 32768
67702024-02-04T14:03:18.025Z20/30 Read block 372 len 4096
67712024-02-04T14:03:18.025Z21/30 Write block 405 len 16384 data: 5 5 5 5
67722024-02-04T14:03:18.025Z22/30 Read block 393 len 16384
67732024-02-04T14:03:18.028Z23/30 Read block 488 len 12288
67742024-02-04T14:03:18.031Z24/30 Write block 248 len 20480 data: 7 9 11 10 10
67752024-02-04T14:03:18.031Z25/30 Flush
67762024-02-04T14:03:18.035Z26/30 Flush
67772024-02-04T14:03:18.038Z27/30 Write block 17 len 28672 data: 4 4 4 4 3 4 4
67782024-02-04T14:03:18.038Z28/30 Write block 515 len 16384 data: 9 10 9 8
67792024-02-04T14:03:18.041Z29/30 Read block 455 len 8192
67802024-02-04T14:03:18.041Z30/30 Read block 462 len 36864
67812024-02-04T14:03:18.045Z----------------------------------------------------------------
67822024-02-04T14:03:18.045Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
67832024-02-04T14:03:18.045ZStates: Active Active Active
67842024-02-04T14:03:18.045ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67852024-02-04T14:03:18.045Z 1 Acked 1000 Write 1 Done Done Done false
67862024-02-04T14:03:18.045Z 2 Acked 1001 Write 4 Done Done Done false
67872024-02-04T14:03:18.045Z 3 Acked 1002 Write 5 Done Done Done false
67882024-02-04T14:03:18.045Z 4 Acked 1003 Write 7 Done Done Done false
67892024-02-04T14:03:18.045Z 5 Acked 1004 Read 9 Done Done Done false
67902024-02-04T14:03:18.045Z 6 Acked 1005 Write 2 Done Done Done false
67912024-02-04T14:03:18.045Z 7 Acked 1006 Read 7 Done Done Done false
67922024-02-04T14:03:18.045Z 8 Acked 1007 Read 3 Done Done Done false
67932024-02-04T14:03:18.045Z 9 Acked 1008 Write 1 Done Done Done false
67942024-02-04T14:03:18.045Z 10 Acked 1009 Read 7 Done Sent Done false
67952024-02-04T14:03:18.045Z 11 Acked 1010 Write 6 Done Sent Done false
67962024-02-04T14:03:18.045Z 12 Acked 1011 Write 4 Done Sent Done false
67972024-02-04T14:03:18.045Z 13 Acked 1012 Write 9 Done Sent Done false
67982024-02-04T14:03:18.045Z 14 Acked 1013 Write 6 Done Sent Done false
67992024-02-04T14:03:18.046Z 15 Acked 1014 Write 2 Done Sent Done false
68002024-02-04T14:03:18.046Z 16 Acked 1015 Read 9 Done Sent Done false
68012024-02-04T14:03:18.046Z 17 Acked 1016 Read 10 Done Sent Done false
68022024-02-04T14:03:18.046Z 18 Acked 1017 Read 5 Done Sent Done false
68032024-02-04T14:03:18.046Z 19 Acked 1018 Read 8 Done Sent Done false
68042024-02-04T14:03:18.046Z 20 Acked 1019 Read 1 Done Sent Done false
68052024-02-04T14:03:18.046Z 21 Acked 1020 Write 4 Done Sent Done false
68062024-02-04T14:03:18.046Z 22 Acked 1021 Read 4 Done Sent Done false
68072024-02-04T14:03:18.046Z 23 Acked 1022 Read 3 Done Sent Done false
68082024-02-04T14:03:18.046Z 24 Acked 1023 Write 5 Done Sent Done false
68092024-02-04T14:03:18.046Z 25 Acked 1024 Flush 0 Done Sent Done false
68102024-02-04T14:03:18.046Z 26 Acked 1025 Flush 0 Done Sent Done false
68112024-02-04T14:03:18.046Z 27 Acked 1026 Write 7 Done Sent Done false
68122024-02-04T14:03:18.046Z 28 Acked 1027 Write 4 Done Sent Done false
68132024-02-04T14:03:18.046Z 29 Acked 1028 Read 2 Done Sent Done false
68142024-02-04T14:03:18.046Z 30 Acked 1029 Read 9 Done Sent Sent false
68152024-02-04T14:03:18.046Z STATES DS:0 DS:1 DS:2 TOTAL
68162024-02-04T14:03:18.046Z New 0 0 0 0
68172024-02-04T14:03:18.046Z Sent 0 21 1 22
68182024-02-04T14:03:18.046Z Done 30 9 29 68
68192024-02-04T14:03:18.046Z Skipped 0 0 0 0
68202024-02-04T14:03:18.046Z Error 0 0 0 0
68212024-02-04T14:03:18.046ZLast Flush: 1025 0 1025
68222024-02-04T14:03:18.046ZDownstairs last five completed:
68232024-02-04T14:03:18.047ZUpstairs last five completed: 30 29 28 27 26
68242024-02-04T14:03:18.047Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:17.076246337Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1377}
68252024-02-04T14:03:18.047ZWrote out file "/tmp/test_repair_verify.data"
68262024-02-04T14:03:18.050Z
68272024-02-04T14:03:18.055ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68282024-02-04T14:03:18.079ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68292024-02-04T14:03:18.079Z 0 000-019 54 54 54 87 87 87 T F T <---
68302024-02-04T14:03:18.079Z 1 020-039 58 58 58 92 92 92 T F T <---
68312024-02-04T14:03:18.079Z 2 040-059 62 61 62 98 97 98 F T F <---
68322024-02-04T14:03:18.079Z 3 060-079 62 61 62 98 97 98 F T F <---
68332024-02-04T14:03:18.079Z 4 080-099 60 60 60 96 96 96 F F F
68342024-02-04T14:03:18.079Z 5 100-119 60 60 60 95 95 95 F F F
68352024-02-04T14:03:18.079Z 6 120-139 51 51 51 78 78 78 F F F
68362024-02-04T14:03:18.083Z 7 140-159 58 58 58 92 92 92 F F F
68372024-02-04T14:03:18.083Z 8 160-179 62 54 62 98 87 98 F F F <---
68382024-02-04T14:03:18.083Z 9 180-199 62 60 62 98 96 98 F F F <---
68392024-02-04T14:03:18.083Z 10 200-219 62 60 62 98 96 98 F F F <---
68402024-02-04T14:03:18.083Z 11 220-239 62 60 62 98 95 98 F T F <---
68412024-02-04T14:03:18.083Z 12 240-259 62 60 62 98 96 98 F F F <---
68422024-02-04T14:03:18.083Z 13 260-279 62 60 62 98 95 98 F F F <---
68432024-02-04T14:03:18.083Z 14 280-299 60 60 60 95 95 95 F F F
68442024-02-04T14:03:18.083Z 15 300-319 58 58 58 92 92 92 F F F
68452024-02-04T14:03:18.083Z 16 320-339 58 58 58 92 92 92 F F F
68462024-02-04T14:03:18.083Z 17 340-359 61 61 61 97 97 97 F F F
68472024-02-04T14:03:18.083Z 18 360-379 56 56 56 89 89 89 F F F
68482024-02-04T14:03:18.083Z 19 380-399 59 59 59 94 94 94 F F F
68492024-02-04T14:03:18.083Z 20 400-419 62 61 62 98 97 98 F T F <---
68502024-02-04T14:03:18.083Z 21 420-439 60 60 60 95 95 95 F F F
68512024-02-04T14:03:18.083Z 22 440-459 62 60 62 98 96 98 F T F <---
68522024-02-04T14:03:18.083Z 23 460-479 60 60 60 96 96 96 F F F
68532024-02-04T14:03:18.083Z 24 480-499 62 57 62 98 91 98 F T F <---
68542024-02-04T14:03:18.083Z 25 500-519 62 60 62 98 95 98 T T T <---
68552024-02-04T14:03:18.083Z 26 520-539 58 58 58 93 93 93 F F F
68562024-02-04T14:03:18.083Z 27 540-559 58 58 58 93 93 93 F F F
68572024-02-04T14:03:18.084Z 28 560-579 58 58 58 93 93 93 F F F
68582024-02-04T14:03:18.084Z 29 580-599 60 60 60 95 95 95 F F F
68592024-02-04T14:03:18.084ZMax gen: 62, Max flush: 98
68602024-02-04T14:03:18.084ZError: Difference in extent metadata found!
68612024-02-04T14:03:18.101ZA Difference in extent metadata is expected here
68622024-02-04T14:03:18.101ZOn loop 31
68632024-02-04T14:03:18.101Z
68642024-02-04T14:03:18.101ZVerifying data now
68652024-02-04T14:03:24.287ZNo change after verify
68662024-02-04T14:03:24.287ZLoop: 31 Downstairs dump after verify (and repair):
68672024-02-04T14:03:24.310ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68682024-02-04T14:03:24.310Z 0 000-019 63 63 63 99 99 99 F F F
68692024-02-04T14:03:24.310Z 1 020-039 63 63 63 99 99 99 F F F
68702024-02-04T14:03:24.310Z 2 040-059 62 62 62 98 98 98 F F F
68712024-02-04T14:03:24.310Z 3 060-079 62 62 62 98 98 98 F F F
68722024-02-04T14:03:24.310Z 4 080-099 60 60 60 96 96 96 F F F
68732024-02-04T14:03:24.310Z 5 100-119 60 60 60 95 95 95 F F F
68742024-02-04T14:03:24.310Z 6 120-139 51 51 51 78 78 78 F F F
68752024-02-04T14:03:24.314Z 7 140-159 58 58 58 92 92 92 F F F
68762024-02-04T14:03:24.314Z 8 160-179 62 62 62 98 98 98 F F F
68772024-02-04T14:03:24.314Z 9 180-199 62 62 62 98 98 98 F F F
68782024-02-04T14:03:24.315Z 10 200-219 62 62 62 98 98 98 F F F
68792024-02-04T14:03:24.315Z 11 220-239 62 62 62 98 98 98 F F F
68802024-02-04T14:03:24.315Z 12 240-259 62 62 62 98 98 98 F F F
68812024-02-04T14:03:24.315Z 13 260-279 62 62 62 98 98 98 F F F
68822024-02-04T14:03:24.315Z 14 280-299 60 60 60 95 95 95 F F F
68832024-02-04T14:03:24.315Z 15 300-319 58 58 58 92 92 92 F F F
68842024-02-04T14:03:24.315Z 16 320-339 58 58 58 92 92 92 F F F
68852024-02-04T14:03:24.315Z 17 340-359 61 61 61 97 97 97 F F F
68862024-02-04T14:03:24.315Z 18 360-379 56 56 56 89 89 89 F F F
68872024-02-04T14:03:24.315Z 19 380-399 59 59 59 94 94 94 F F F
68882024-02-04T14:03:24.315Z 20 400-419 62 62 62 98 98 98 F F F
68892024-02-04T14:03:24.315Z 21 420-439 60 60 60 95 95 95 F F F
68902024-02-04T14:03:24.315Z 22 440-459 62 62 62 98 98 98 F F F
68912024-02-04T14:03:24.315Z 23 460-479 60 60 60 96 96 96 F F F
68922024-02-04T14:03:24.315Z 24 480-499 62 62 62 98 98 98 F F F
68932024-02-04T14:03:24.315Z 25 500-519 63 63 63 99 99 99 F F F
68942024-02-04T14:03:24.315Z 26 520-539 58 58 58 93 93 93 F F F
68952024-02-04T14:03:24.316Z 27 540-559 58 58 58 93 93 93 F F F
68962024-02-04T14:03:24.316Z 28 560-579 58 58 58 93 93 93 F F F
68972024-02-04T14:03:24.316Z 29 580-599 60 60 60 95 95 95 F F F
68982024-02-04T14:03:24.316ZMax gen: 63, Max flush: 99
68992024-02-04T14:03:24.319Z
69002024-02-04T14:03:24.319ZBegin loop 32
69012024-02-04T14:03:24.319ZDownstairs to restart: 2
69022024-02-04T14:03:24.346Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.379161838Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385}
69032024-02-04T14:03:24.346Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.379618381Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385}
69042024-02-04T14:03:24.346Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.379648607Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385}
69052024-02-04T14:03:24.350Z{"msg":"Crucible 2e9428d7-0a93-498b-9b12-6471b520384d has session id: 912108e9-28bf-4b8a-b217-39d16bc07941","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.379745592Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69062024-02-04T14:03:24.350Z{"msg":"Upstairs opts: Upstairs UUID: 2e9428d7-0a93-498b-9b12-6471b520384d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
69072024-02-04T14:03:24.350Z,"time":"2024-02-04T14:03:23.379831275Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69082024-02-04T14:03:24.350Z{"msg":"Crucible stats registered with UUID: 2e9428d7-0a93-498b-9b12-6471b520384d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.379875453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69092024-02-04T14:03:24.350Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.379949292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385}
69102024-02-04T14:03:24.350Z{"msg":"2e9428d7-0a93-498b-9b12-6471b520384d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380290722Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69112024-02-04T14:03:24.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380346439Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69122024-02-04T14:03:24.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380372291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69132024-02-04T14:03:24.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380432932Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69142024-02-04T14:03:24.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380459515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69152024-02-04T14:03:24.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380519981Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69162024-02-04T14:03:24.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380547714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69172024-02-04T14:03:24.350Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380570131Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"io task","client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69182024-02-04T14:03:24.350Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.38059316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"io task","client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69192024-02-04T14:03:24.351Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380639201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"io task","client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69202024-02-04T14:03:24.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.380934296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"io task","client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69212024-02-04T14:03:24.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.381087397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"io task","client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69222024-02-04T14:03:24.351Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.381119777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"io task","client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69232024-02-04T14:03:24.351Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.382325622Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69242024-02-04T14:03:24.351Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.382415964Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69252024-02-04T14:03:24.351Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.382577948Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69262024-02-04T14:03:24.351Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.38269538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69272024-02-04T14:03:24.355Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.383087105Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69282024-02-04T14:03:24.355Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.383204886Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69292024-02-04T14:03:24.355Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.383411322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69302024-02-04T14:03:24.355Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.383558044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69312024-02-04T14:03:24.355Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.384299912Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69322024-02-04T14:03:24.356Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.384379842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69332024-02-04T14:03:24.356Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.384426028Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69342024-02-04T14:03:24.356Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.384506968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69352024-02-04T14:03:24.356Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.384578951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69362024-02-04T14:03:24.356Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.384657823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69372024-02-04T14:03:24.356Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.384734392Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69382024-02-04T14:03:24.356Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385243022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69392024-02-04T14:03:24.356Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.38528787Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69402024-02-04T14:03:24.356Z{"msg":"[0]R flush_numbers[0..12]: [99, 99, 98, 98, 96, 95, 78, 92, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385314356Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69412024-02-04T14:03:24.356Z{"msg":"[0]R generation[0..12]: [63, 63, 62, 62, 60, 60, 51, 58, 62, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.38533862Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69422024-02-04T14:03:24.356Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385386956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69432024-02-04T14:03:24.356Z{"msg":"[1]R flush_numbers[0..12]: [99, 99, 98, 98, 96, 95, 78, 92, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385413763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69442024-02-04T14:03:24.357Z{"msg":"[1]R generation[0..12]: [63, 63, 62, 62, 60, 60, 51, 58, 62, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385456308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69452024-02-04T14:03:24.357Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385482092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69462024-02-04T14:03:24.357Z{"msg":"[2]R flush_numbers[0..12]: [99, 99, 98, 98, 96, 95, 78, 92, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385560764Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69472024-02-04T14:03:24.357Z{"msg":"[2]R generation[0..12]: [63, 63, 62, 62, 60, 60, 51, 58, 62, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385586626Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69482024-02-04T14:03:24.357Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]Wait for a query_work_queue command to finish before sending IO
69492024-02-04T14:03:24.357Z","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385614118Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69502024-02-04T14:03:24.357Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.38566453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69512024-02-04T14:03:24.357Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385688461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69522024-02-04T14:03:24.357Z{"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385790192Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69532024-02-04T14:03:24.357Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385813917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69542024-02-04T14:03:24.357Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385836169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69552024-02-04T14:03:24.357Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385875177Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69562024-02-04T14:03:24.357Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385898951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69572024-02-04T14:03:24.358Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385944856Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69582024-02-04T14:03:24.358Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.385968689Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69592024-02-04T14:03:24.358Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.386008703Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69602024-02-04T14:03:24.358Z{"msg":"2e9428d7-0a93-498b-9b12-6471b520384d is now active with session: 912108e9-28bf-4b8a-b217-39d16bc07941","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.386032128Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69612024-02-04T14:03:24.358Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.386074775Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385,"session_id":"912108e9-28bf-4b8a-b217-39d16bc07941"}
69622024-02-04T14:03:24.358Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:23.38609682Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385}
69632024-02-04T14:03:24.358ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69642024-02-04T14:03:24.358ZLoading write count information from file "/tmp/test_repair_verify.data"
69652024-02-04T14:03:24.358ZRun Repair workload
69662024-02-04T14:03:24.358Z01/30 Write block 201 len 16384 data: 9 9 9 8
69672024-02-04T14:03:24.362Z02/30 Read block 136 len 24576
69682024-02-04T14:03:24.362Z03/30 Read block 307 len 32768
69692024-02-04T14:03:24.362Z04/30 Write block 305 len 32768 data: 5 4 5 4 6 6 7 6
69702024-02-04T14:03:24.362Z05/30 Read block 564 len 32768
69712024-02-04T14:03:24.377Z06/30 Read block 523 len 40960
69722024-02-04T14:03:24.377Z07/30 Read block 42 len 16384
69732024-02-04T14:03:24.381Z08/30 Read block 266 len 8192
69742024-02-04T14:03:24.381Z09/30 Read block 579 len 32768
69752024-02-04T14:03:24.381Z10/30 Read block 67 len 28672
69762024-02-04T14:03:24.381Z11/30 Read block 64 len 40960
69772024-02-04T14:03:24.381Z12/30 Flush
69782024-02-04T14:03:24.381Z13/30 Write block 15 len 36864 data: 5 5 5 5 5 5 4 5 5
69792024-02-04T14:03:24.381Z14/30 Write block 132 len 40960 data: 2 4 4 4 2 2 2 2 5 4
69802024-02-04T14:03:24.381Z15/30 Read block 87 len 28672
69812024-02-04T14:03:24.384Z16/30 Read block 469 len 24576
69822024-02-04T14:03:24.385Z17/30 Write block 319 len 36864 data: 6 6 5 5 5 4 4 4 4
69832024-02-04T14:03:24.385Z18/30 Read block 418 len 20480
69842024-02-04T14:03:24.388Z19/30 Write block 419 len 8192 data: 7 10
69852024-02-04T14:03:24.388Z20/30 Read block 72 len 12288
69862024-02-04T14:03:24.393Z21/30 Read block 178 len 12288
69872024-02-04T14:03:24.393Z22/30 Write block 584 len 20480 data: 10 9 8 7 6
69882024-02-04T14:03:24.393Z23/30 Read block 161 len 28672
69892024-02-04T14:03:24.396Z24/30 Read block 411 len 28672
69902024-02-04T14:03:24.396Z25/30 Write block 538 len 12288 data: 5 5 6
69912024-02-04T14:03:24.396Z26/30 Flush
69922024-02-04T14:03:24.400Z27/30 Write block 526 len 8192 data: 4 3
69932024-02-04T14:03:24.403Z28/30 Write block 261 len 32768 data: 6 8 8 7 6 7 6 6
69942024-02-04T14:03:24.403Z29/30 Write block 385 len 4096 data: 5
69952024-02-04T14:03:24.403Z30/30 Read block 45 len 36864
69962024-02-04T14:03:24.410Z----------------------------------------------------------------
69972024-02-04T14:03:24.410Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69982024-02-04T14:03:24.410ZStates: Active Active Active
69992024-02-04T14:03:24.410ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
70002024-02-04T14:03:24.410Z 1 Acked 1000 Write 4 Done Done Done false
70012024-02-04T14:03:24.411Z 2 Acked 1001 Read 6 Done Done Done false
70022024-02-04T14:03:24.411Z 3 Acked 1002 Read 8 Done Done Done false
70032024-02-04T14:03:24.411Z 4 Acked 1003 Write 8 Done Done Done false
70042024-02-04T14:03:24.411Z 5 Acked 1004 Read 8 Done Done Sent false
70052024-02-04T14:03:24.411Z 6 Acked 1005 Read 10 Done Done Sent false
70062024-02-04T14:03:24.411Z 7 Acked 1006 Read 4 Done Done Sent false
70072024-02-04T14:03:24.411Z 8 Acked 1007 Read 2 Done Done Sent false
70082024-02-04T14:03:24.411Z 9 Acked 1008 Read 8 Done Done Sent false
70092024-02-04T14:03:24.411Z 10 Acked 1009 Read 7 Done Done Sent false
70102024-02-04T14:03:24.411Z 11 Acked 1010 Read 10 Done Done Sent false
70112024-02-04T14:03:24.411Z 12 Acked 1011 Flush 0 Done Done Sent false
70122024-02-04T14:03:24.411Z 13 Acked 1012 Write 9 Done Done Sent false
70132024-02-04T14:03:24.412Z 14 Acked 1013 Write 10 Done Done Sent false
70142024-02-04T14:03:24.412Z 15 Acked 1014 Read 7 Done Done Sent false
70152024-02-04T14:03:24.412Z 16 Acked 1015 Read 6 Done Done Sent false
70162024-02-04T14:03:24.412Z 17 Acked 1016 Write 9 Done Done Sent false
70172024-02-04T14:03:24.412Z 18 Acked 1017 Read 5 Done Done Sent false
70182024-02-04T14:03:24.412Z 19 Acked 1018 Write 2 Done Done Sent false
70192024-02-04T14:03:24.412Z 20 Acked 1019 Read 3 Done Done Sent false
70202024-02-04T14:03:24.412Z 21 Acked 1020 Read 3 Done Done Sent false
70212024-02-04T14:03:24.412Z 22 Acked 1021 Write 5 Done Done Sent false
70222024-02-04T14:03:24.412Z 23 Acked 1022 Read 7 Done Done Sent false
70232024-02-04T14:03:24.412Z 24 Acked 1023 Read 7 Done Done Sent false
70242024-02-04T14:03:24.412Z 25 Acked 1024 Write 3 Done Done Sent false
70252024-02-04T14:03:24.412Z 26 Acked 1025 Flush 0 Done Done Sent false
70262024-02-04T14:03:24.413Z 27 Acked 1026 Write 2 Done Done Sent false
70272024-02-04T14:03:24.413Z 28 Acked 1027 Write 8 Done Done Sent false
70282024-02-04T14:03:24.413Z 29 Acked 1028 Write 1 Done Done Sent false
70292024-02-04T14:03:24.413Z 30 Acked 1029 Read 9 Sent Done Sent false
70302024-02-04T14:03:24.413Z STATES DS:0 DS:1 DS:2 TOTAL
70312024-02-04T14:03:24.413Z New 0 0 0 0
70322024-02-04T14:03:24.413Z Sent 1 0 26 27
70332024-02-04T14:03:24.413Z Done 29 30 4 63
70342024-02-04T14:03:24.413Z Skipped 0 0 0 0
70352024-02-04T14:03:24.413Z Error 0 0 0 0
70362024-02-04T14:03:24.413ZLast Flush: 1025 1025 0
70372024-02-04T14:03:24.413ZDownstairs last five completed:
70382024-02-04T14:03:24.413ZUpstairs last five completed: 30 29 28 27 26
70392024-02-04T14:03:24.414Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:23.438512337Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1385}
70402024-02-04T14:03:24.414ZWrote out file "/tmp/test_repair_verify.data"
70412024-02-04T14:03:24.417Z
70422024-02-04T14:03:24.421ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70432024-02-04T14:03:24.444ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70442024-02-04T14:03:24.444Z 0 000-019 64 64 63 101 101 99 F F F <---
70452024-02-04T14:03:24.444Z 1 020-039 64 64 63 101 101 99 F F F <---
70462024-02-04T14:03:24.444Z 2 040-059 62 62 62 98 98 98 F F F
70472024-02-04T14:03:24.444Z 3 060-079 62 62 62 98 98 98 F F F
70482024-02-04T14:03:24.444Z 4 080-099 60 60 60 96 96 96 F F F
70492024-02-04T14:03:24.444Z 5 100-119 60 60 60 95 95 95 F F F
70502024-02-04T14:03:24.448Z 6 120-139 64 64 51 101 101 78 F F F <---
70512024-02-04T14:03:24.448Z 7 140-159 64 64 58 101 101 92 F F F <---
70522024-02-04T14:03:24.448Z 8 160-179 62 62 62 98 98 98 F F F
70532024-02-04T14:03:24.448Z 9 180-199 62 62 62 98 98 98 F F F
70542024-02-04T14:03:24.448Z 10 200-219 64 64 62 100 100 98 F F T <---
70552024-02-04T14:03:24.448Z 11 220-239 62 62 62 98 98 98 F F F
70562024-02-04T14:03:24.448Z 12 240-259 62 62 62 98 98 98 F F F
70572024-02-04T14:03:24.448Z 13 260-279 62 62 62 98 98 98 T T F <---
70582024-02-04T14:03:24.448Z 14 280-299 60 60 60 95 95 95 F F F
70592024-02-04T14:03:24.449Z 15 300-319 64 64 58 101 101 92 F F T <---
70602024-02-04T14:03:24.449Z 16 320-339 64 64 58 101 101 92 F F F <---
70612024-02-04T14:03:24.449Z 17 340-359 61 61 61 97 97 97 F F F
70622024-02-04T14:03:24.449Z 18 360-379 56 56 56 89 89 89 F F F
70632024-02-04T14:03:24.449Z 19 380-399 59 59 59 94 94 94 T T F <---
70642024-02-04T14:03:24.449Z 20 400-419 64 64 62 101 101 98 F F F <---
70652024-02-04T14:03:24.449Z 21 420-439 64 64 60 101 101 95 F F F <---
70662024-02-04T14:03:24.449Z 22 440-459 62 62 62 98 98 98 F F F
70672024-02-04T14:03:24.449Z 23 460-479 60 60 60 96 96 96 F F F
70682024-02-04T14:03:24.449Z 24 480-499 62 62 62 98 98 98 F F F
70692024-02-04T14:03:24.449Z 25 500-519 63 63 63 99 99 99 F F F
70702024-02-04T14:03:24.449Z 26 520-539 64 64 58 101 101 93 T T F <---
70712024-02-04T14:03:24.449Z 27 540-559 64 64 58 101 101 93 F F F <---
70722024-02-04T14:03:24.449Z 28 560-579 58 58 58 93 93 93 F F F
70732024-02-04T14:03:24.449Z 29 580-599 64 64 60 101 101 95 F F F <---
70742024-02-04T14:03:24.449ZMax gen: 64, Max flush: 101
70752024-02-04T14:03:24.449ZError: Difference in extent metadata found!
70762024-02-04T14:03:24.453ZA Difference in extent metadata is expected here
70772024-02-04T14:03:24.453ZOn loop 32
70782024-02-04T14:03:24.453Z
70792024-02-04T14:03:24.453ZVerifying data now
70802024-02-04T14:03:30.642ZNo change after verify
70812024-02-04T14:03:30.642ZLoop: 32 Downstairs dump after verify (and repair):
70822024-02-04T14:03:30.666ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70832024-02-04T14:03:30.666Z 0 000-019 64 64 64 101 101 101 F F F
70842024-02-04T14:03:30.666Z 1 020-039 64 64 64 101 101 101 F F F
70852024-02-04T14:03:30.666Z 2 040-059 62 62 62 98 98 98 F F F
70862024-02-04T14:03:30.666Z 3 060-079 62 62 62 98 98 98 F F F
70872024-02-04T14:03:30.666Z 4 080-099 60 60 60 96 96 96 F F F
70882024-02-04T14:03:30.666Z 5 100-119 60 60 60 95 95 95 F F F
70892024-02-04T14:03:30.667Z 6 120-139 64 64 64 101 101 101 F F F
70902024-02-04T14:03:30.667Z 7 140-159 64 64 64 101 101 101 F F F
70912024-02-04T14:03:30.667Z 8 160-179 62 62 62 98 98 98 F F F
70922024-02-04T14:03:30.667Z 9 180-199 62 62 62 98 98 98 F F F
70932024-02-04T14:03:30.667Z 10 200-219 64 64 64 100 100 100 F F F
70942024-02-04T14:03:30.667Z 11 220-239 62 62 62 98 98 98 F F F
70952024-02-04T14:03:30.667Z 12 240-259 62 62 62 98 98 98 F F F
70962024-02-04T14:03:30.667Z 13 260-279 65 65 65 102 102 102 F F F
70972024-02-04T14:03:30.667Z 14 280-299 60 60 60 95 95 95 F F F
70982024-02-04T14:03:30.667Z 15 300-319 64 64 64 101 101 101 F F F
70992024-02-04T14:03:30.673Z 16 320-339 64 64 64 101 101 101 F F F
71002024-02-04T14:03:30.673Z 17 340-359 61 61 61 97 97 97 F F F
71012024-02-04T14:03:30.673Z 18 360-379 56 56 56 89 89 89 F F F
71022024-02-04T14:03:30.673Z 19 380-399 65 65 65 102 102 102 F F F
71032024-02-04T14:03:30.673Z 20 400-419 64 64 64 101 101 101 F F F
71042024-02-04T14:03:30.673Z 21 420-439 64 64 64 101 101 101 F F F
71052024-02-04T14:03:30.673Z 22 440-459 62 62 62 98 98 98 F F F
71062024-02-04T14:03:30.673Z 23 460-479 60 60 60 96 96 96 F F F
71072024-02-04T14:03:30.673Z 24 480-499 62 62 62 98 98 98 F F F
71082024-02-04T14:03:30.673Z 25 500-519 63 63 63 99 99 99 F F F
71092024-02-04T14:03:30.673Z 26 520-539 65 65 65 102 102 102 F F F
71102024-02-04T14:03:30.674Z 27 540-559 64 64 64 101 101 101 F F F
71112024-02-04T14:03:30.674Z 28 560-579 58 58 58 93 93 93 F F F
71122024-02-04T14:03:30.674Z 29 580-599 64 64 64 101 101 101 F F F
71132024-02-04T14:03:30.674ZMax gen: 65, Max flush: 102
71142024-02-04T14:03:30.677Z
71152024-02-04T14:03:30.678ZBegin loop 33
71162024-02-04T14:03:30.678ZDownstairs to restart: 1
71172024-02-04T14:03:30.706Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.738749403Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393}
71182024-02-04T14:03:30.706Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739232265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393}
71192024-02-04T14:03:30.706Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739274314Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393}
71202024-02-04T14:03:30.710Z{"msg":"Crucible c2f54b2a-543a-401a-aed0-06d9f820f612 has session id: 2c629aff-32a8-4529-95a0-5c75822cfbdc","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739367398Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71212024-02-04T14:03:30.710Z{"msg":"Upstairs opts: Upstairs UUID: c2f54b2a-543a-401a-aed0-06d9f820f612, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739419876Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71222024-02-04T14:03:30.710Z{"msg":"Crucible stats registered with UUID: c2f54b2a-543a-401a-aed0-06d9f820f612","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739500988Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393Crucible runtime is spawned
71232024-02-04T14:03:30.710Z,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71242024-02-04T14:03:30.710Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739711293Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393}
71252024-02-04T14:03:30.710Z{"msg":"c2f54b2a-543a-401a-aed0-06d9f820f612 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739777919Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71262024-02-04T14:03:30.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739821527Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71272024-02-04T14:03:30.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739907105Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71282024-02-04T14:03:30.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739941525Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71292024-02-04T14:03:30.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739965514Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71302024-02-04T14:03:30.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.739988955Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71312024-02-04T14:03:30.711Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.740052784Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71322024-02-04T14:03:30.711Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.740081456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71332024-02-04T14:03:30.711Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.74010826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71342024-02-04T14:03:30.711Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.740160198Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71352024-02-04T14:03:30.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.740435061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71362024-02-04T14:03:30.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.740516576Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71372024-02-04T14:03:30.711Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.740590023Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71382024-02-04T14:03:30.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.741783012Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71392024-02-04T14:03:30.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.741849597Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71402024-02-04T14:03:30.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.741875151Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71412024-02-04T14:03:30.711Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.741897852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71422024-02-04T14:03:30.717Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.74290227Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71432024-02-04T14:03:30.717Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.743205069Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71442024-02-04T14:03:30.717Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.743424718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71452024-02-04T14:03:30.717Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.743489824Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71462024-02-04T14:03:30.717Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.743804062Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71472024-02-04T14:03:30.717Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.743894683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71482024-02-04T14:03:30.718Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.743939683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71492024-02-04T14:03:30.718Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.743997862Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71502024-02-04T14:03:30.718Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.744025685Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71512024-02-04T14:03:30.718Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.744049361Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71522024-02-04T14:03:30.718Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.744408321Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71532024-02-04T14:03:30.718Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745148481Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71542024-02-04T14:03:30.718Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745208229Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71552024-02-04T14:03:30.718Z{"msg":"[0]R flush_numbers[0..12]: [101, 101, 98, 98, 96, 95, 101, 101, 98, 98, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745234519Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71562024-02-04T14:03:30.718Z{"msg":"[0]R generation[0..12]: [64, 64, 62, 62, 60, 60, 64, 64, 62, 62, 64, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745258658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71572024-02-04T14:03:30.718Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745302879Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71582024-02-04T14:03:30.718Z{"msg":"[1]R flush_numbers[0..12]: [101, 101, 98, 98, 96, 95, 101, 101, 98, 98, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745327677Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71592024-02-04T14:03:30.718Z{"msg":"[1]R generation[0..12]: [64, 64, 62, 62, 60, 60, 64, 64, 62, 62, 64, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745369669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71602024-02-04T14:03:30.719Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745395056Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71612024-02-04T14:03:30.719Z{"msg":"[2]R flush_numbers[0..12]: [101, 101, 98, 98, 96, 95, 101, 101, 98, 98, 100, 98]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
71622024-02-04T14:03:30.719Z,"time":"2024-02-04T14:03:29.745437287Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71632024-02-04T14:03:30.719Z{"msg":"[2]R generation[0..12]: [64, 64, 62, 62, 60, 60, 64, 64, 62, 62, 64, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745462767Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71642024-02-04T14:03:30.719Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745510472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71652024-02-04T14:03:30.719Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745534201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71662024-02-04T14:03:30.719Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745575086Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71672024-02-04T14:03:30.719Z{"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745598411Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71682024-02-04T14:03:30.719Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745640429Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71692024-02-04T14:03:30.719Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745664204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71702024-02-04T14:03:30.719Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745703205Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71712024-02-04T14:03:30.719Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745726973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71722024-02-04T14:03:30.720Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745749052Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71732024-02-04T14:03:30.720Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745793974Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71742024-02-04T14:03:30.720Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745816157Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71752024-02-04T14:03:30.720Z{"msg":"c2f54b2a-543a-401a-aed0-06d9f820f612 is now active with session: 2c629aff-32a8-4529-95a0-5c75822cfbdc","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.74583797Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71762024-02-04T14:03:30.720Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745883787Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393,"session_id":"2c629aff-32a8-4529-95a0-5c75822cfbdc"}
71772024-02-04T14:03:30.720Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:29.745905936Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393}
71782024-02-04T14:03:30.720ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71792024-02-04T14:03:30.720ZLoading write count information from file "/tmp/test_repair_verify.data"
71802024-02-04T14:03:30.720ZRun Repair workload
71812024-02-04T14:03:30.720Z01/30 Write block 499 len 32768 data: 9 9 9 6 6 7 7 8
71822024-02-04T14:03:30.724Z02/30 Write block 71 len 28672 data: 5 5 5 6 6 6 7
71832024-02-04T14:03:30.724Z03/30 Write block 305 len 16384 data: 6 5 6 5
71842024-02-04T14:03:30.724Z04/30 Read block 419 len 12288
71852024-02-04T14:03:30.724Z05/30 Write block 219 len 12288 data: 5 5 6
71862024-02-04T14:03:30.725Z06/30 Write block 186 len 16384 data: 6 5 5 5
71872024-02-04T14:03:30.725Z07/30 Read block 474 len 28672
71882024-02-04T14:03:30.732Z08/30 Read block 157 len 28672
71892024-02-04T14:03:30.732Z09/30 Write block 467 len 16384 data: 9 7 8 8
71902024-02-04T14:03:30.732Z10/30 Read block 37 len 4096
71912024-02-04T14:03:30.735Z11/30 Write block 402 len 24576 data: 6 4 4 6 6 6
71922024-02-04T14:03:30.735Z12/30 Read block 514 len 28672
71932024-02-04T14:03:30.735Z13/30 Write block 507 len 28672 data: 8 7 8 7 6 4 4
71942024-02-04T14:03:30.735Z14/30 Write block 341 len 32768 data: 5 4 4 4 4 5 5 3
71952024-02-04T14:03:30.736Z15/30 Read block 192 len 36864
71962024-02-04T14:03:30.739Z16/30 Read block 322 len 24576
71972024-02-04T14:03:30.739Z17/30 Read block 572 len 4096
71982024-02-04T14:03:30.739Z18/30 Flush
71992024-02-04T14:03:30.747Z19/30 Write block 387 len 4096 data: 6
72002024-02-04T14:03:30.750Z20/30 Write block 293 len 32768 data: 4 4 4 3 3 4 5 3
72012024-02-04T14:03:30.750Z21/30 Read block 181 len 16384
72022024-02-04T14:03:30.750Z22/30 Flush
72032024-02-04T14:03:30.753Z23/30 Write block 391 len 16384 data: 6 7 8 8
72042024-02-04T14:03:30.753Z24/30 Write block 170 len 4096 data: 8
72052024-02-04T14:03:30.753Z25/30 Write block 354 len 4096 data: 8
72062024-02-04T14:03:30.757Z26/30 Write block 155 len 4096 data: 4
72072024-02-04T14:03:30.757Z27/30 Read block 39 len 36864
72082024-02-04T14:03:30.757Z28/30 Read block 161 len 24576
72092024-02-04T14:03:30.761Z29/30 Write block 71 len 32768 data: 6 6 6 7 7 7 8 6
72102024-02-04T14:03:30.761Z30/30 Write block 434 len 16384 data: 6 6 5 5
72112024-02-04T14:03:30.761Z----------------------------------------------------------------
72122024-02-04T14:03:30.761Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
72132024-02-04T14:03:30.761ZStates: Active Active Active
72142024-02-04T14:03:30.761ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72152024-02-04T14:03:30.761Z 1 Acked 1000 Write 8 Done Done Done false
72162024-02-04T14:03:30.761Z 2 Acked 1001 Write 7 Done Sent Done false
72172024-02-04T14:03:30.761Z 3 Acked 1002 Write 4 Done Sent Done false
72182024-02-04T14:03:30.761Z 4 Acked 1003 Read 3 Done Sent Done false
72192024-02-04T14:03:30.761Z 5 Acked 1004 Write 3 Done Sent Done false
72202024-02-04T14:03:30.761Z 6 Acked 1005 Write 4 Done Sent Done false
72212024-02-04T14:03:30.762Z 7 Acked 1006 Read 7 Done Sent Done false
72222024-02-04T14:03:30.762Z 8 Acked 1007 Read 7 Done Sent Done false
72232024-02-04T14:03:30.762Z 9 Acked 1008 Write 4 Done Sent Done false
72242024-02-04T14:03:30.762Z 10 Acked 1009 Read 1 Done Sent Done false
72252024-02-04T14:03:30.762Z 11 Acked 1010 Write 6 Done Sent Done false
72262024-02-04T14:03:30.762Z 12 Acked 1011 Read 7 Done Sent Done false
72272024-02-04T14:03:30.762Z 13 Acked 1012 Write 7 Done Sent Done false
72282024-02-04T14:03:30.762Z 14 Acked 1013 Write 8 Done Sent Done false
72292024-02-04T14:03:30.762Z 15 Acked 1014 Read 9 Done Sent Done false
72302024-02-04T14:03:30.762Z 16 Acked 1015 Read 6 Done Sent Done false
72312024-02-04T14:03:30.762Z 17 Acked 1016 Read 1 Done Sent Done false
72322024-02-04T14:03:30.762Z 18 Acked 1017 Flush 0 Done Sent Done false
72332024-02-04T14:03:30.763Z 19 Acked 1018 Write 1 Done Sent Done false
72342024-02-04T14:03:30.763Z 20 Acked 1019 Write 8 Done Sent Done false
72352024-02-04T14:03:30.763Z 21 Acked 1020 Read 4 Done Sent Done false
72362024-02-04T14:03:30.763Z 22 Acked 1021 Flush 0 Done Sent Done false
72372024-02-04T14:03:30.763Z 23 Acked 1022 Write 4 Done Sent Done false
72382024-02-04T14:03:30.763Z 24 Acked 1023 Write 1 Done Sent Done false
72392024-02-04T14:03:30.763Z 25 Acked 1024 Write 1 Done Sent Done false
72402024-02-04T14:03:30.763Z 26 Acked 1025 Write 1 Done Sent Done false
72412024-02-04T14:03:30.763Z 27 Acked 1026 Read 9 Done Sent Done false
72422024-02-04T14:03:30.763Z 28 Acked 1027 Read 6 Done Sent Done false
72432024-02-04T14:03:30.763Z 29 Acked 1028 Write 8 Sent Sent Sent false
72442024-02-04T14:03:30.763Z 30 Acked 1029 Write 4 Sent Sent Sent false
72452024-02-04T14:03:30.763Z STATES DS:0 DS:1 DS:2 TOTAL
72462024-02-04T14:03:30.764Z New 0 0 0 0
72472024-02-04T14:03:30.764Z Sent 2 29 2 33
72482024-02-04T14:03:30.764Z Done 28 1 28 57
72492024-02-04T14:03:30.764Z Skipped 0 0 0 0
72502024-02-04T14:03:30.764Z Error 0 0 0 0
72512024-02-04T14:03:30.764ZLast Flush: 1021 0 1021
72522024-02-04T14:03:30.764ZDownstairs last five completed:
72532024-02-04T14:03:30.764ZUpstairs last five completed: 30 29 28 27 26
72542024-02-04T14:03:30.768Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:29.793613042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1393}
72552024-02-04T14:03:30.768ZWrote out file "/tmp/test_repair_verify.data"
72562024-02-04T14:03:30.771Z
72572024-02-04T14:03:30.775ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72582024-02-04T14:03:30.804ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72592024-02-04T14:03:30.805Z 0 000-019 64 64 64 101 101 101 F F F
72602024-02-04T14:03:30.805Z 1 020-039 64 64 64 101 101 101 F F F
72612024-02-04T14:03:30.805Z 2 040-059 62 62 62 98 98 98 F F F
72622024-02-04T14:03:30.805Z 3 060-079 66 62 66 103 98 103 T F T <---
72632024-02-04T14:03:30.805Z 4 080-099 60 60 60 96 96 96 F F F
72642024-02-04T14:03:30.805Z 5 100-119 60 60 60 95 95 95 F F F
72652024-02-04T14:03:30.805Z 6 120-139 64 64 64 101 101 101 F F F
72662024-02-04T14:03:30.805Z 7 140-159 64 64 64 101 101 101 T F T <---
72672024-02-04T14:03:30.805Z 8 160-179 62 62 62 98 98 98 T F T <---
72682024-02-04T14:03:30.805Z 9 180-199 66 62 66 103 98 103 F F F <---
72692024-02-04T14:03:30.809Z 10 200-219 66 64 66 103 100 103 F F F <---
72702024-02-04T14:03:30.809Z 11 220-239 66 62 66 103 98 103 F F F <---
72712024-02-04T14:03:30.810Z 12 240-259 62 62 62 98 98 98 F F F
72722024-02-04T14:03:30.810Z 13 260-279 65 65 65 102 102 102 F F F
72732024-02-04T14:03:30.810Z 14 280-299 66 60 66 104 95 104 F F F <---
72742024-02-04T14:03:30.810Z 15 300-319 66 64 66 104 101 104 F F F <---
72752024-02-04T14:03:30.810Z 16 320-339 64 64 64 101 101 101 F F F
72762024-02-04T14:03:30.810Z 17 340-359 66 61 66 103 97 103 T F T <---
72772024-02-04T14:03:30.810Z 18 360-379 56 56 56 89 89 89 F F F
72782024-02-04T14:03:30.810Z 19 380-399 66 65 66 104 102 104 T F T <---
72792024-02-04T14:03:30.810Z 20 400-419 66 64 66 103 101 103 F F F <---
72802024-02-04T14:03:30.810Z 21 420-439 64 64 64 101 101 101 T F T <---
72812024-02-04T14:03:30.810Z 22 440-459 62 62 62 98 98 98 F F F
72822024-02-04T14:03:30.810Z 23 460-479 66 60 66 103 96 103 F F F <---
72832024-02-04T14:03:30.811Z 24 480-499 66 62 66 103 98 103 F T F <---
72842024-02-04T14:03:30.811Z 25 500-519 66 63 66 103 99 103 F T F <---
72852024-02-04T14:03:30.811Z 26 520-539 65 65 65 102 102 102 F F F
72862024-02-04T14:03:30.811Z 27 540-559 64 64 64 101 101 101 F F F
72872024-02-04T14:03:30.811Z 28 560-579 58 58 58 93 93 93 F F F
72882024-02-04T14:03:30.811Z 29 580-599 64 64 64 101 101 101 F F F
72892024-02-04T14:03:30.811ZMax gen: 66, Max flush: 104
72902024-02-04T14:03:30.811ZError: Difference in extent metadata found!
72912024-02-04T14:03:30.815ZA Difference in extent metadata is expected here
72922024-02-04T14:03:30.815ZOn loop 33
72932024-02-04T14:03:30.815Z
72942024-02-04T14:03:30.819ZVerifying data now
72952024-02-04T14:03:37.141ZNo change after verify
72962024-02-04T14:03:37.141ZLoop: 33 Downstairs dump after verify (and repair):
72972024-02-04T14:03:37.162ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72982024-02-04T14:03:37.163Z 0 000-019 64 64 64 101 101 101 F F F
72992024-02-04T14:03:37.163Z 1 020-039 64 64 64 101 101 101 F F F
73002024-02-04T14:03:37.163Z 2 040-059 62 62 62 98 98 98 F F F
73012024-02-04T14:03:37.163Z 3 060-079 67 67 67 105 105 105 F F F
73022024-02-04T14:03:37.163Z 4 080-099 60 60 60 96 96 96 F F F
73032024-02-04T14:03:37.163Z 5 100-119 60 60 60 95 95 95 F F F
73042024-02-04T14:03:37.163Z 6 120-139 64 64 64 101 101 101 F F F
73052024-02-04T14:03:37.163Z 7 140-159 67 67 67 105 105 105 F F F
73062024-02-04T14:03:37.163Z 8 160-179 67 67 67 105 105 105 F F F
73072024-02-04T14:03:37.163Z 9 180-199 66 66 66 103 103 103 F F F
73082024-02-04T14:03:37.163Z 10 200-219 66 66 66 103 103 103 F F F
73092024-02-04T14:03:37.163Z 11 220-239 66 66 66 103 103 103 F F F
73102024-02-04T14:03:37.166Z 12 240-259 62 62 62 98 98 98 F F F
73112024-02-04T14:03:37.166Z 13 260-279 65 65 65 102 102 102 F F F
73122024-02-04T14:03:37.166Z 14 280-299 66 66 66 104 104 104 F F F
73132024-02-04T14:03:37.166Z 15 300-319 66 66 66 104 104 104 F F F
73142024-02-04T14:03:37.167Z 16 320-339 64 64 64 101 101 101 F F F
73152024-02-04T14:03:37.167Z 17 340-359 67 67 67 105 105 105 F F F
73162024-02-04T14:03:37.167Z 18 360-379 56 56 56 89 89 89 F F F
73172024-02-04T14:03:37.167Z 19 380-399 67 67 67 105 105 105 F F F
73182024-02-04T14:03:37.167Z 20 400-419 66 66 66 103 103 103 F F F
73192024-02-04T14:03:37.167Z 21 420-439 67 67 67 105 105 105 F F F
73202024-02-04T14:03:37.167Z 22 440-459 62 62 62 98 98 98 F F F
73212024-02-04T14:03:37.167Z 23 460-479 66 66 66 103 103 103 F F F
73222024-02-04T14:03:37.167Z 24 480-499 66 66 66 103 103 103 F F F
73232024-02-04T14:03:37.167Z 25 500-519 66 66 66 103 103 103 F F F
73242024-02-04T14:03:37.167Z 26 520-539 65 65 65 102 102 102 F F F
73252024-02-04T14:03:37.167Z 27 540-559 64 64 64 101 101 101 F F F
73262024-02-04T14:03:37.167Z 28 560-579 58 58 58 93 93 93 F F F
73272024-02-04T14:03:37.167Z 29 580-599 64 64 64 101 101 101 F F F
73282024-02-04T14:03:37.167ZMax gen: 67, Max flush: 105
73292024-02-04T14:03:37.170Z
73302024-02-04T14:03:37.170ZBegin loop 34
73312024-02-04T14:03:37.170ZDownstairs to restart: 1
73322024-02-04T14:03:37.201Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.233057095Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401}
73332024-02-04T14:03:37.201Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.23350898Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401}
73342024-02-04T14:03:37.205Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.233579725Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401}
73352024-02-04T14:03:37.205ZCrucible runtime is spawned
73362024-02-04T14:03:37.205Z{"msg":"Crucible b144cab3-cbc2-48cf-96e4-ad97efc0f3c5 has session id: 149c2563-a79c-4330-9a55-a20a5ee8248d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.233807564Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73372024-02-04T14:03:37.205Z{"msg":"Upstairs opts: Upstairs UUID: b144cab3-cbc2-48cf-96e4-ad97efc0f3c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.233915131Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73382024-02-04T14:03:37.205Z{"msg":"Crucible stats registered with UUID: b144cab3-cbc2-48cf-96e4-ad97efc0f3c5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.233958141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73392024-02-04T14:03:37.205Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.233995488Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401}
73402024-02-04T14:03:37.205Z{"msg":"b144cab3-cbc2-48cf-96e4-ad97efc0f3c5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234141784Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73412024-02-04T14:03:37.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.23421903Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73422024-02-04T14:03:37.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234266001Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73432024-02-04T14:03:37.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234304651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73442024-02-04T14:03:37.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234371911Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73452024-02-04T14:03:37.206Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234421195Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73462024-02-04T14:03:37.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234460061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73472024-02-04T14:03:37.206Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234532954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"io task","client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73482024-02-04T14:03:37.206Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.23464472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"io task","client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73492024-02-04T14:03:37.206Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234695443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"io task","client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73502024-02-04T14:03:37.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234740111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"io task","client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73512024-02-04T14:03:37.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234783375Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"io task","client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73522024-02-04T14:03:37.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.234826004Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"io task","client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73532024-02-04T14:03:37.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.235691946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73542024-02-04T14:03:37.206Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.235764015Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73552024-02-04T14:03:37.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.235808293Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73562024-02-04T14:03:37.206Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.235850765Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73572024-02-04T14:03:37.206Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.236729074Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73582024-02-04T14:03:37.206Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.236851568Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73592024-02-04T14:03:37.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.23773505Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73602024-02-04T14:03:37.206Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.23788631Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73612024-02-04T14:03:37.206Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.237989472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73622024-02-04T14:03:37.206Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.238087629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73632024-02-04T14:03:37.210Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.23811383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73642024-02-04T14:03:37.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.238295914Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73652024-02-04T14:03:37.210Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.238348656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73662024-02-04T14:03:37.211Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.238372913Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73672024-02-04T14:03:37.211Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.238745883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73682024-02-04T14:03:37.211Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239532693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73692024-02-04T14:03:37.211Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239629344Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73702024-02-04T14:03:37.211Z{"msg":"[0]R flush_numbers[0..12]: [101, 101, 98, 105, 96, 95, 101, 105, 105, 103, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239654508Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73712024-02-04T14:03:37.211Z{"msg":"[0]R generation[0..12]: [64, 64, 62, 67, 60, 60, 64, 67, 67, 66, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239712458Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73722024-02-04T14:03:37.211Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239738374Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73732024-02-04T14:03:37.211Z{"msg":"[1]R flush_numbers[0..12]: [101, 101, 98, 105, 96, 95, 101, 105, 105, 103, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239780534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73742024-02-04T14:03:37.211Z{"msg":"[1]R generation[0..12]: [64, 64, 62, 67, 60, 60, 64, 67, 67, 66, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239804808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73752024-02-04T14:03:37.211Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239846884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73762024-02-04T14:03:37.211Z{"msg":"[2]R flush_numbers[0..12]: [101, 101, 98, 105, 96, 95, 101, 105, 105, 103, 103, 103]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
73772024-02-04T14:03:37.211Z,"time":"2024-02-04T14:03:36.239871562Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73782024-02-04T14:03:37.211Z{"msg":"[2]R generation[0..12]: [64, 64, 62, 67, 60, 60, 64, 67, 67, 66, 66, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239895138Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73792024-02-04T14:03:37.211Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239940324Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73802024-02-04T14:03:37.211Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239963748Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73812024-02-04T14:03:37.211Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.239986899Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73822024-02-04T14:03:37.211Z{"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240008615Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73832024-02-04T14:03:37.211Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240030904Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73842024-02-04T14:03:37.211Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240081373Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73852024-02-04T14:03:37.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240103142Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73862024-02-04T14:03:37.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240125345Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73872024-02-04T14:03:37.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.24017285Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73882024-02-04T14:03:37.211Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240196306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73892024-02-04T14:03:37.212Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240217386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73902024-02-04T14:03:37.212Z{"msg":"b144cab3-cbc2-48cf-96e4-ad97efc0f3c5 is now active with session: 149c2563-a79c-4330-9a55-a20a5ee8248d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240239438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73912024-02-04T14:03:37.212Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240283646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401,"session_id":"149c2563-a79c-4330-9a55-a20a5ee8248d"}
73922024-02-04T14:03:37.212Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:36.240305475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401}
73932024-02-04T14:03:37.212ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73942024-02-04T14:03:37.212ZLoading write count information from file "/tmp/test_repair_verify.data"
73952024-02-04T14:03:37.212ZRun Repair workload
73962024-02-04T14:03:37.212Z01/30 Write block 284 len 24576 data: 10 9 7 8 9 8
73972024-02-04T14:03:37.215Z02/30 Flush
73982024-02-04T14:03:37.215Z03/30 Write block 216 len 28672 data: 8 8 6 6 6 7 5
73992024-02-04T14:03:37.215Z04/30 Read block 481 len 36864
74002024-02-04T14:03:37.218Z05/30 Read block 546 len 12288
74012024-02-04T14:03:37.221Z06/30 Write block 399 len 28672 data: 8 8 7 7 5 5 7
74022024-02-04T14:03:37.221Z07/30 Read block 53 len 20480
74032024-02-04T14:03:37.224Z08/30 Read block 10 len 36864
74042024-02-04T14:03:37.224Z09/30 Read block 596 len 12288
74052024-02-04T14:03:37.227Z10/30 Write block 390 len 36864 data: 4 7 8 9 9 6 6 8 7
74062024-02-04T14:03:37.227Z11/30 Read block 539 len 12288
74072024-02-04T14:03:37.230Z12/30 Read block 446 len 12288
74082024-02-04T14:03:37.230Z13/30 Write block 97 len 40960 data: 5 5 6 4 4 3 2 4 4 5
74092024-02-04T14:03:37.233Z14/30 Read block 295 len 12288
74102024-02-04T14:03:37.233Z15/30 Read block 365 len 40960
74112024-02-04T14:03:37.236Z16/30 Write block 589 len 8192 data: 2 3
74122024-02-04T14:03:37.236Z17/30 Read block 193 len 24576
74132024-02-04T14:03:37.242Z18/30 Write block 389 len 40960 data: 7 5 8 9 10 10 7 7 9 8
74142024-02-04T14:03:37.242Z19/30 Read block 114 len 8192
74152024-02-04T14:03:37.247Z20/30 Write block 200 len 32768 data: 9 10 10 10 9 7 6 5
74162024-02-04T14:03:37.247Z21/30 Read block 523 len 28672
74172024-02-04T14:03:37.247Z22/30 Read block 159 len 40960
74182024-02-04T14:03:37.251Z23/30 Write block 574 len 32768 data: 7 7 9 9 8 10 12 10
74192024-02-04T14:03:37.251Z24/30 Write block 144 len 32768 data: 5 6 6 5 5 6 6 6
74202024-02-04T14:03:37.252Z25/30 Flush
74212024-02-04T14:03:37.255Z26/30 Write block 400 len 12288 data: 9 8 8
74222024-02-04T14:03:37.258Z27/30 Write block 588 len 32768 data: 7 3 4 3 4 4 4 4
74232024-02-04T14:03:37.258Z28/30 Write block 185 len 12288 data: 9 7 6
74242024-02-04T14:03:37.258Z29/30 Read block 324 len 20480
74252024-02-04T14:03:37.262Z30/30 Read block 175 len 36864
74262024-02-04T14:03:37.262Z----------------------------------------------------------------
74272024-02-04T14:03:37.262Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:28
74282024-02-04T14:03:37.262ZStates: Active Active Active
74292024-02-04T14:03:37.262ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74302024-02-04T14:03:37.262Z 3 Acked 1002 Write 7 Done Done Done false
74312024-02-04T14:03:37.262Z 4 Acked 1003 Read 9 Done Done Done false
74322024-02-04T14:03:37.262Z 5 Acked 1004 Read 3 Done Done Done false
74332024-02-04T14:03:37.262Z 6 Acked 1005 Write 7 Done Done Done false
74342024-02-04T14:03:37.262Z 7 Acked 1006 Read 5 Done Sent Done false
74352024-02-04T14:03:37.262Z 8 Acked 1007 Read 9 Done Sent Done false
74362024-02-04T14:03:37.262Z 9 Acked 1008 Read 3 Done Sent Done false
74372024-02-04T14:03:37.262Z 10 Acked 1009 Write 9 Done Sent Done false
74382024-02-04T14:03:37.262Z 11 Acked 1010 Read 3 Done Sent Done false
74392024-02-04T14:03:37.262Z 12 Acked 1011 Read 3 Done Sent Done false
74402024-02-04T14:03:37.262Z 13 Acked 1012 Write 10 Done Sent Done false
74412024-02-04T14:03:37.262Z 14 Acked 1013 Read 3 Done Sent Done false
74422024-02-04T14:03:37.262Z 15 Acked 1014 Read 10 Done Sent Done false
74432024-02-04T14:03:37.262Z 16 Acked 1015 Write 2 Done Sent Done false
74442024-02-04T14:03:37.263Z 17 Acked 1016 Read 6 Done Sent Done false
74452024-02-04T14:03:37.263Z 18 Acked 1017 Write 10 Done Sent Done false
74462024-02-04T14:03:37.263Z 19 Acked 1018 Read 2 Done Sent Done false
74472024-02-04T14:03:37.263Z 20 Acked 1019 Write 8 Done Sent Done false
74482024-02-04T14:03:37.263Z 21 Acked 1020 Read 7 Done Sent Done false
74492024-02-04T14:03:37.263Z 22 Acked 1021 Read 10 Done Sent Done false
74502024-02-04T14:03:37.263Z 23 Acked 1022 Write 8 Done Sent Done false
74512024-02-04T14:03:37.263Z 24 Acked 1023 Write 8 Done Sent Done false
74522024-02-04T14:03:37.263Z 25 Acked 1024 Flush 0 Done Sent Done false
74532024-02-04T14:03:37.263Z 26 Acked 1025 Write 3 Done Sent Done false
74542024-02-04T14:03:37.263Z 27 Acked 1026 Write 8 Done Sent Done false
74552024-02-04T14:03:37.263Z 28 Acked 1027 Write 3 Done Sent Done false
74562024-02-04T14:03:37.263Z 29 Acked 1028 Read 5 Done Sent Done false
74572024-02-04T14:03:37.263Z 30 Acked 1029 Read 9 Done Sent Done false
74582024-02-04T14:03:37.263Z STATES DS:0 DS:1 DS:2 TOTAL
74592024-02-04T14:03:37.263Z New 0 0 0 0
74602024-02-04T14:03:37.263Z Sent 0 24 0 24
74612024-02-04T14:03:37.263Z Done 28 4 28 60
74622024-02-04T14:03:37.263Z Skipped 0 0 0 0
74632024-02-04T14:03:37.263Z Error 0 0 0 0
74642024-02-04T14:03:37.263ZLast Flush: 1024 1001 1024
74652024-02-04T14:03:37.263ZDownstairs last five completed: 1001 1000
74662024-02-04T14:03:37.263ZUpstairs last five completed: 30 29 28 27 26
74672024-02-04T14:03:37.263Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:36.294263315Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1401}
74682024-02-04T14:03:37.263ZWrote out file "/tmp/test_repair_verify.data"
74692024-02-04T14:03:37.268Z
74702024-02-04T14:03:37.275ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74712024-02-04T14:03:37.301ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74722024-02-04T14:03:37.301Z 0 000-019 64 64 64 101 101 101 F F F
74732024-02-04T14:03:37.301Z 1 020-039 64 64 64 101 101 101 F F F
74742024-02-04T14:03:37.301Z 2 040-059 62 62 62 98 98 98 F F F
74752024-02-04T14:03:37.301Z 3 060-079 67 67 67 105 105 105 F F F
74762024-02-04T14:03:37.301Z 4 080-099 68 60 68 107 96 107 F F F <---
74772024-02-04T14:03:37.301Z 5 100-119 68 60 68 107 95 107 F F F <---
74782024-02-04T14:03:37.301Z 6 120-139 64 64 64 101 101 101 F F F
74792024-02-04T14:03:37.301Z 7 140-159 68 67 68 107 105 107 F F F <---
74802024-02-04T14:03:37.302Z 8 160-179 67 67 67 105 105 105 F F F
74812024-02-04T14:03:37.302Z 9 180-199 66 66 66 103 103 103 T F T <---
74822024-02-04T14:03:37.302Z 10 200-219 68 66 68 107 103 107 F T F <---
74832024-02-04T14:03:37.302Z 11 220-239 68 66 68 107 103 107 F T F <---
74842024-02-04T14:03:37.305Z 12 240-259 62 62 62 98 98 98 F F F
74852024-02-04T14:03:37.305Z 13 260-279 65 65 65 102 102 102 F F F
74862024-02-04T14:03:37.305Z 14 280-299 68 68 68 106 106 106 F F F
74872024-02-04T14:03:37.305Z 15 300-319 66 66 66 104 104 104 F F F
74882024-02-04T14:03:37.305Z 16 320-339 64 64 64 101 101 101 F F F
74892024-02-04T14:03:37.305Z 17 340-359 67 67 67 105 105 105 F F F
74902024-02-04T14:03:37.305Z 18 360-379 56 56 56 89 89 89 F F F
74912024-02-04T14:03:37.305Z 19 380-399 68 67 68 107 105 107 F T F <---
74922024-02-04T14:03:37.305Z 20 400-419 68 66 68 107 103 107 T T T <---
74932024-02-04T14:03:37.305Z 21 420-439 67 67 67 105 105 105 F F F
74942024-02-04T14:03:37.305Z 22 440-459 62 62 62 98 98 98 F F F
74952024-02-04T14:03:37.305Z 23 460-479 66 66 66 103 103 103 F F F
74962024-02-04T14:03:37.306Z 24 480-499 66 66 66 103 103 103 F F F
74972024-02-04T14:03:37.306Z 25 500-519 66 66 66 103 103 103 F F F
74982024-02-04T14:03:37.306Z 26 520-539 65 65 65 102 102 102 F F F
74992024-02-04T14:03:37.306Z 27 540-559 64 64 64 101 101 101 F F F
75002024-02-04T14:03:37.306Z 28 560-579 68 58 68 107 93 107 F F F <---
75012024-02-04T14:03:37.306Z 29 580-599 68 64 68 107 101 107 T F T <---
75022024-02-04T14:03:37.306ZMax gen: 68, Max flush: 107
75032024-02-04T14:03:37.306ZError: Difference in extent metadata found!
75042024-02-04T14:03:37.309ZA Difference in extent metadata is expected here
75052024-02-04T14:03:37.309ZOn loop 34
75062024-02-04T14:03:37.309Z
75072024-02-04T14:03:37.313ZVerifying data now
75082024-02-04T14:03:43.240ZNo change after verify
75092024-02-04T14:03:43.240ZLoop: 34 Downstairs dump after verify (and repair):
75102024-02-04T14:03:43.262ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75112024-02-04T14:03:43.263Z 0 000-019 64 64 64 101 101 101 F F F
75122024-02-04T14:03:43.263Z 1 020-039 64 64 64 101 101 101 F F F
75132024-02-04T14:03:43.263Z 2 040-059 62 62 62 98 98 98 F F F
75142024-02-04T14:03:43.263Z 3 060-079 67 67 67 105 105 105 F F F
75152024-02-04T14:03:43.266Z 4 080-099 68 68 68 107 107 107 F F F
75162024-02-04T14:03:43.266Z 5 100-119 68 68 68 107 107 107 F F F
75172024-02-04T14:03:43.266Z 6 120-139 64 64 64 101 101 101 F F F
75182024-02-04T14:03:43.266Z 7 140-159 68 68 68 107 107 107 F F F
75192024-02-04T14:03:43.266Z 8 160-179 67 67 67 105 105 105 F F F
75202024-02-04T14:03:43.266Z 9 180-199 69 69 69 108 108 108 F F F
75212024-02-04T14:03:43.266Z 10 200-219 68 68 68 107 107 107 F F F
75222024-02-04T14:03:43.266Z 11 220-239 68 68 68 107 107 107 F F F
75232024-02-04T14:03:43.266Z 12 240-259 62 62 62 98 98 98 F F F
75242024-02-04T14:03:43.266Z 13 260-279 65 65 65 102 102 102 F F F
75252024-02-04T14:03:43.266Z 14 280-299 68 68 68 106 106 106 F F F
75262024-02-04T14:03:43.266Z 15 300-319 66 66 66 104 104 104 F F F
75272024-02-04T14:03:43.266Z 16 320-339 64 64 64 101 101 101 F F F
75282024-02-04T14:03:43.266Z 17 340-359 67 67 67 105 105 105 F F F
75292024-02-04T14:03:43.266Z 18 360-379 56 56 56 89 89 89 F F F
75302024-02-04T14:03:43.266Z 19 380-399 68 68 68 107 107 107 F F F
75312024-02-04T14:03:43.266Z 20 400-419 69 69 69 108 108 108 F F F
75322024-02-04T14:03:43.266Z 21 420-439 67 67 67 105 105 105 F F F
75332024-02-04T14:03:43.266Z 22 440-459 62 62 62 98 98 98 F F F
75342024-02-04T14:03:43.267Z 23 460-479 66 66 66 103 103 103 F F F
75352024-02-04T14:03:43.267Z 24 480-499 66 66 66 103 103 103 F F F
75362024-02-04T14:03:43.267Z 25 500-519 66 66 66 103 103 103 F F F
75372024-02-04T14:03:43.267Z 26 520-539 65 65 65 102 102 102 F F F
75382024-02-04T14:03:43.267Z 27 540-559 64 64 64 101 101 101 F F F
75392024-02-04T14:03:43.267Z 28 560-579 68 68 68 107 107 107 F F F
75402024-02-04T14:03:43.267Z 29 580-599 69 69 69 108 108 108 F F F
75412024-02-04T14:03:43.267ZMax gen: 69, Max flush: 108
75422024-02-04T14:03:43.270Z
75432024-02-04T14:03:43.270ZBegin loop 35
75442024-02-04T14:03:43.270ZDownstairs to restart: 1
75452024-02-04T14:03:43.299Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.331955256Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409}
75462024-02-04T14:03:43.299Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.332499842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409}
75472024-02-04T14:03:43.303Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.332572984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409}
75482024-02-04T14:03:43.303Z{"msg":"Crucible f89efce3-7581-46cb-8c04-4241eb5e9117 has session id: 03794129-5e4d-4a45-b3d5-17aca73ad318","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.332642405Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75492024-02-04T14:03:43.303Z{"msg":"Upstairs opts: Upstairs UUID: f89efce3-7581-46cb-8c04-4241eb5e9117, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.332689344Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75502024-02-04T14:03:43.303Z{"msg":"Crucible stats registered with UUID: f89efce3-7581-46cb-8c04-4241eb5e9117","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.332720209Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75512024-02-04T14:03:43.303ZCrucible runtime is spawned
75522024-02-04T14:03:43.303Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.33298118Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409}
75532024-02-04T14:03:43.303Z{"msg":"f89efce3-7581-46cb-8c04-4241eb5e9117 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.333127751Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75542024-02-04T14:03:43.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.33317801Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75552024-02-04T14:03:43.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.333248475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75562024-02-04T14:03:43.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.333294991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75572024-02-04T14:03:43.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.333368388Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75582024-02-04T14:03:43.303Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.333413163Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75592024-02-04T14:03:43.303Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.33348843Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75602024-02-04T14:03:43.303Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.333591039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75612024-02-04T14:03:43.303Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.333718676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75622024-02-04T14:03:43.303Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.33392266Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75632024-02-04T14:03:43.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.334008938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75642024-02-04T14:03:43.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.334053728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75652024-02-04T14:03:43.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.334127802Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75662024-02-04T14:03:43.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.335239835Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75672024-02-04T14:03:43.304Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.335445901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75682024-02-04T14:03:43.304Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.335547399Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75692024-02-04T14:03:43.304Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.335613314Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75702024-02-04T14:03:43.308Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.336299181Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75712024-02-04T14:03:43.308Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.336510299Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75722024-02-04T14:03:43.308Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.336598146Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75732024-02-04T14:03:43.308Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.336744398Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75742024-02-04T14:03:43.308Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.337266979Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75752024-02-04T14:03:43.308Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.337331904Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75762024-02-04T14:03:43.308Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.337485499Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75772024-02-04T14:03:43.308Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.337521636Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75782024-02-04T14:03:43.308Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.33758202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75792024-02-04T14:03:43.308Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.337609119Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75802024-02-04T14:03:43.308Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.338041871Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75812024-02-04T14:03:43.308Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.338768235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75822024-02-04T14:03:43.308Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.338844494Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75832024-02-04T14:03:43.308Z{"msg":"[0]R flush_numbers[0..12]: [101, 101, 98, 105, 107, 107, 101, 107, 105, 108, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.338891927Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75842024-02-04T14:03:43.308Z{"msg":"[0]R generation[0..12]: [64, 64, 62, 67, 68, 68, 64, 68, 67, 69, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.338949732Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75852024-02-04T14:03:43.308Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.338977331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75862024-02-04T14:03:43.308Z{"msg":"[1]R flush_numbers[0..12]: [101, 101, 98, 105, 107, 107, 101, 107, 105, 108, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339032215Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75872024-02-04T14:03:43.308Z{"msg":"[1]R generation[0..12]: [64, 64, 62, 67, 68, 68, 64, 68, 67, 69, 68, 68]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
75882024-02-04T14:03:43.308Zlevel":30,"time":"2024-02-04T14:03:42.339188343Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75892024-02-04T14:03:43.309Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339261272Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75902024-02-04T14:03:43.309Z{"msg":"[2]R flush_numbers[0..12]: [101, 101, 98, 105, 107, 107, 101, 107, 105, 108, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339289433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75912024-02-04T14:03:43.309Z{"msg":"[2]R generation[0..12]: [64, 64, 62, 67, 68, 68, 64, 68, 67, 69, 68, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339332202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75922024-02-04T14:03:43.309Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339368575Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75932024-02-04T14:03:43.309Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339434471Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75942024-02-04T14:03:43.309Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.33950729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75952024-02-04T14:03:43.309Z{"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.3395718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75962024-02-04T14:03:43.309Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339598403Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75972024-02-04T14:03:43.309Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339654725Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75982024-02-04T14:03:43.309Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339695072Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
75992024-02-04T14:03:43.309Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339749487Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
76002024-02-04T14:03:43.309Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.339790784Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
76012024-02-04T14:03:43.309Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76022024-02-04T14:03:43.309Z:30,"time":"2024-02-04T14:03:42.339933783Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
76032024-02-04T14:03:43.309Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.340008764Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
76042024-02-04T14:03:43.309Z{"msg":"f89efce3-7581-46cb-8c04-4241eb5e9117 is now active with session: 03794129-5e4d-4a45-b3d5-17aca73ad318","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.340077747Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
76052024-02-04T14:03:43.309Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.340144543Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409,"session_id":"03794129-5e4d-4a45-b3d5-17aca73ad318"}
76062024-02-04T14:03:43.309Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:42.340211317Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409}
76072024-02-04T14:03:43.309ZLoading write count information from file "/tmp/test_repair_verify.data"
76082024-02-04T14:03:43.309ZRun Repair workload
76092024-02-04T14:03:43.314Z01/30 Write block 506 len 12288 data: 9 9 8
76102024-02-04T14:03:43.314Z02/30 Write block 258 len 16384 data: 6 6 4 7
76112024-02-04T14:03:43.314Z03/30 Write block 198 len 28672 data: 8 9 10 11 11 11 10
76122024-02-04T14:03:43.314Z04/30 Read block 79 len 24576
76132024-02-04T14:03:43.318Z05/30 Read block 425 len 8192
76142024-02-04T14:03:43.318Z06/30 Write block 588 len 40960 data: 8 4 5 4 5 5 5 5 2 2
76152024-02-04T14:03:43.321Z07/30 Read block 382 len 24576
76162024-02-04T14:03:43.321Z08/30 Read block 58 len 28672
76172024-02-04T14:03:43.324Z09/30 Flush
76182024-02-04T14:03:43.327Z10/30 Write block 565 len 8192 data: 7 7
76192024-02-04T14:03:43.327Z11/30 Write block 283 len 20480 data: 7 11 10 8 9
76202024-02-04T14:03:43.330Z12/30 Read block 48 len 24576
76212024-02-04T14:03:43.330Z13/30 Read block 385 len 40960
76222024-02-04T14:03:43.333Z14/30 Read block 282 len 4096
76232024-02-04T14:03:43.333Z15/30 Write block 172 len 40960 data: 10 10 10 9 8 8 7 9 9 9
76242024-02-04T14:03:43.335Z16/30 Read block 136 len 12288
76252024-02-04T14:03:43.335Z17/30 Write block 0 len 40960 data: 2 2 3 3 5 5 5 5 5 6
76262024-02-04T14:03:43.338Z18/30 Read block 47 len 12288
76272024-02-04T14:03:43.338Z19/30 Read block 489 len 12288
76282024-02-04T14:03:43.341Z20/30 Read block 357 len 20480
76292024-02-04T14:03:43.341Z21/30 Write block 343 len 12288 data: 5 5 5
76302024-02-04T14:03:43.344Z22/30 Write block 166 len 20480 data: 7 7 7 7 9
76312024-02-04T14:03:43.344Z23/30 Read block 408 len 20480
76322024-02-04T14:03:43.344Z24/30 Write block 284 len 4096 data: 12
76332024-02-04T14:03:43.347Z25/30 Write block 440 len 32768 data: 4 5 4 4 4 4 4 4
76342024-02-04T14:03:43.347Z26/30 Flush
76352024-02-04T14:03:43.351Z27/30 Write block 483 len 28672 data: 2 3 5 5 5 6 8
76362024-02-04T14:03:43.357Z28/30 Write block 366 len 20480 data: 5 6 5 5 5
76372024-02-04T14:03:43.357Z29/30 Read block 84 len 40960
76382024-02-04T14:03:43.360Z30/30 Write block 348 len 24576 data: 4 6 4 6 8 9
76392024-02-04T14:03:43.360Z----------------------------------------------------------------
76402024-02-04T14:03:43.360Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
76412024-02-04T14:03:43.360ZStates: Active Active Active
76422024-02-04T14:03:43.360ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76432024-02-04T14:03:43.360Z 1 Acked 1000 Write 3 Done Done Done false
76442024-02-04T14:03:43.360Z 2 Acked 1001 Write 4 Done Done Done false
76452024-02-04T14:03:43.360Z 3 Acked 1002 Write 7 Done Done Done false
76462024-02-04T14:03:43.361Z 4 Acked 1003 Read 6 Done Done Done false
76472024-02-04T14:03:43.361Z 5 Acked 1004 Read 2 Done Sent Done false
76482024-02-04T14:03:43.361Z 6 Acked 1005 Write 10 Done Sent Done false
76492024-02-04T14:03:43.361Z 7 Acked 1006 Read 6 Done Sent Done false
76502024-02-04T14:03:43.361Z 8 Acked 1007 Read 7 Done Sent Done false
76512024-02-04T14:03:43.361Z 9 Acked 1008 Flush 0 Done Sent Done false
76522024-02-04T14:03:43.361Z 10 Acked 1009 Write 2 Done Sent Done false
76532024-02-04T14:03:43.361Z 11 Acked 1010 Write 5 Done Sent Done false
76542024-02-04T14:03:43.361Z 12 Acked 1011 Read 6 Done Sent Done false
76552024-02-04T14:03:43.361Z 13 Acked 1012 Read 10 Done Sent Done false
76562024-02-04T14:03:43.361Z 14 Acked 1013 Read 1 Done Sent Done false
76572024-02-04T14:03:43.361Z 15 Acked 1014 Write 10 Done Sent Done false
76582024-02-04T14:03:43.361Z 16 Acked 1015 Read 3 Done Sent Done false
76592024-02-04T14:03:43.361Z 17 Acked 1016 Write 10 Done Sent Done false
76602024-02-04T14:03:43.361Z 18 Acked 1017 Read 3 Done Sent Done false
76612024-02-04T14:03:43.361Z 19 Acked 1018 Read 3 Done Sent Done false
76622024-02-04T14:03:43.361Z 20 Acked 1019 Read 5 Done Sent Done false
76632024-02-04T14:03:43.361Z 21 Acked 1020 Write 3 Done Sent Done false
76642024-02-04T14:03:43.361Z 22 Acked 1021 Write 5 Done Sent Done false
76652024-02-04T14:03:43.361Z 23 Acked 1022 Read 5 Done Sent Done false
76662024-02-04T14:03:43.361Z 24 Acked 1023 Write 1 Done Sent Done false
76672024-02-04T14:03:43.361Z 25 Acked 1024 Write 8 Done Sent Done false
76682024-02-04T14:03:43.361Z 26 Acked 1025 Flush 0 Done Sent Done false
76692024-02-04T14:03:43.361Z 27 Acked 1026 Write 7 Done Sent Done false
76702024-02-04T14:03:43.362Z 28 Acked 1027 Write 5 Done Sent Done false
76712024-02-04T14:03:43.362Z 29 Acked 1028 Read 10 Done Sent Done false
76722024-02-04T14:03:43.362Z 30 Acked 1029 Write 6 Sent Sent Sent false
76732024-02-04T14:03:43.362Z STATES DS:0 DS:1 DS:2 TOTAL
76742024-02-04T14:03:43.362Z New 0 0 0 0
76752024-02-04T14:03:43.362Z Sent 1 26 1 28
76762024-02-04T14:03:43.362Z Done 29 4 29 62
76772024-02-04T14:03:43.362Z Skipped 0 0 0 0
76782024-02-04T14:03:43.362Z Error 0 0 0 0
76792024-02-04T14:03:43.362ZLast Flush: 1025 0 1025
76802024-02-04T14:03:43.362ZDownstairs last five completed:
76812024-02-04T14:03:43.362ZUpstairs last five completed: 30 29 28 27 26
76822024-02-04T14:03:43.362Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:42.390659789Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1409}
76832024-02-04T14:03:43.362ZWrote out file "/tmp/test_repair_verify.data"
76842024-02-04T14:03:43.368Z
76852024-02-04T14:03:43.371ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76862024-02-04T14:03:43.397ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76872024-02-04T14:03:43.398Z 0 000-019 70 64 70 110 101 110 F F F <---
76882024-02-04T14:03:43.398Z 1 020-039 64 64 64 101 101 101 F F F
76892024-02-04T14:03:43.398Z 2 040-059 62 62 62 98 98 98 F F F
76902024-02-04T14:03:43.398Z 3 060-079 67 67 67 105 105 105 F F F
76912024-02-04T14:03:43.398Z 4 080-099 68 68 68 107 107 107 F F F
76922024-02-04T14:03:43.398Z 5 100-119 68 68 68 107 107 107 F F F
76932024-02-04T14:03:43.398Z 6 120-139 64 64 64 101 101 101 F F F
76942024-02-04T14:03:43.398Z 7 140-159 68 68 68 107 107 107 F F F
76952024-02-04T14:03:43.398Z 8 160-179 70 67 70 110 105 110 F F F <---
76962024-02-04T14:03:43.398Z 9 180-199 70 69 70 110 108 110 F T F <---
76972024-02-04T14:03:43.398Z 10 200-219 70 68 70 109 107 109 F T F <---
76982024-02-04T14:03:43.398Z 11 220-239 68 68 68 107 107 107 F F F
76992024-02-04T14:03:43.401Z 12 240-259 70 62 70 109 98 109 F T F <---
77002024-02-04T14:03:43.401Z 13 260-279 70 65 70 109 102 109 F T F <---
77012024-02-04T14:03:43.401Z 14 280-299 70 68 70 110 106 110 F F F <---
77022024-02-04T14:03:43.401Z 15 300-319 66 66 66 104 104 104 F F F
77032024-02-04T14:03:43.401Z 16 320-339 64 64 64 101 101 101 F F F
77042024-02-04T14:03:43.401Z 17 340-359 70 67 70 110 105 110 T F T <---
77052024-02-04T14:03:43.401Z 18 360-379 56 56 56 89 89 89 T F T <---
77062024-02-04T14:03:43.401Z 19 380-399 68 68 68 107 107 107 F F F
77072024-02-04T14:03:43.401Z 20 400-419 69 69 69 108 108 108 F F F
77082024-02-04T14:03:43.401Z 21 420-439 67 67 67 105 105 105 F F F
77092024-02-04T14:03:43.401Z 22 440-459 70 62 70 110 98 110 F F F <---
77102024-02-04T14:03:43.401Z 23 460-479 66 66 66 103 103 103 F F F
77112024-02-04T14:03:43.402Z 24 480-499 66 66 66 103 103 103 T F T <---
77122024-02-04T14:03:43.402Z 25 500-519 70 66 70 109 103 109 F T F <---
77132024-02-04T14:03:43.402Z 26 520-539 65 65 65 102 102 102 F F F
77142024-02-04T14:03:43.402Z 27 540-559 64 64 64 101 101 101 F F F
77152024-02-04T14:03:43.402Z 28 560-579 70 68 70 110 107 110 F F F <---
77162024-02-04T14:03:43.402Z 29 580-599 70 69 70 109 108 109 F F F <---
77172024-02-04T14:03:43.402ZMax gen: 70, Max flush: 110
77182024-02-04T14:03:43.402ZError: Difference in extent metadata found!
77192024-02-04T14:03:43.405ZA Difference in extent metadata is expected here
77202024-02-04T14:03:43.405ZOn loop 35
77212024-02-04T14:03:43.405Z
77222024-02-04T14:03:43.408ZVerifying data now
77232024-02-04T14:03:49.626ZNo change after verify
77242024-02-04T14:03:49.626ZLoop: 35 Downstairs dump after verify (and repair):
77252024-02-04T14:03:49.650ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77262024-02-04T14:03:49.650Z 0 000-019 70 70 70 110 110 110 F F F
77272024-02-04T14:03:49.650Z 1 020-039 64 64 64 101 101 101 F F F
77282024-02-04T14:03:49.650Z 2 040-059 62 62 62 98 98 98 F F F
77292024-02-04T14:03:49.650Z 3 060-079 67 67 67 105 105 105 F F F
77302024-02-04T14:03:49.650Z 4 080-099 68 68 68 107 107 107 F F F
77312024-02-04T14:03:49.650Z 5 100-119 68 68 68 107 107 107 F F F
77322024-02-04T14:03:49.650Z 6 120-139 64 64 64 101 101 101 F F F
77332024-02-04T14:03:49.650Z 7 140-159 68 68 68 107 107 107 F F F
77342024-02-04T14:03:49.650Z 8 160-179 70 70 70 110 110 110 F F F
77352024-02-04T14:03:49.650Z 9 180-199 70 70 70 110 110 110 F F F
77362024-02-04T14:03:49.650Z 10 200-219 70 70 70 109 109 109 F F F
77372024-02-04T14:03:49.650Z 11 220-239 68 68 68 107 107 107 F F F
77382024-02-04T14:03:49.650Z 12 240-259 70 70 70 109 109 109 F F F
77392024-02-04T14:03:49.650Z 13 260-279 70 70 70 109 109 109 F F F
77402024-02-04T14:03:49.650Z 14 280-299 70 70 70 110 110 110 F F F
77412024-02-04T14:03:49.650Z 15 300-319 66 66 66 104 104 104 F F F
77422024-02-04T14:03:49.650Z 16 320-339 64 64 64 101 101 101 F F F
77432024-02-04T14:03:49.650Z 17 340-359 71 71 71 111 111 111 F F F
77442024-02-04T14:03:49.650Z 18 360-379 71 71 71 111 111 111 F F F
77452024-02-04T14:03:49.651Z 19 380-399 68 68 68 107 107 107 F F F
77462024-02-04T14:03:49.651Z 20 400-419 69 69 69 108 108 108 F F F
77472024-02-04T14:03:49.651Z 21 420-439 67 67 67 105 105 105 F F F
77482024-02-04T14:03:49.654Z 22 440-459 70 70 70 110 110 110 F F F
77492024-02-04T14:03:49.654Z 23 460-479 66 66 66 103 103 103 F F F
77502024-02-04T14:03:49.654Z 24 480-499 71 71 71 111 111 111 F F F
77512024-02-04T14:03:49.654Z 25 500-519 70 70 70 109 109 109 F F F
77522024-02-04T14:03:49.654Z 26 520-539 65 65 65 102 102 102 F F F
77532024-02-04T14:03:49.654Z 27 540-559 64 64 64 101 101 101 F F F
77542024-02-04T14:03:49.654Z 28 560-579 70 70 70 110 110 110 F F F
77552024-02-04T14:03:49.654Z 29 580-599 70 70 70 109 109 109 F F F
77562024-02-04T14:03:49.654ZMax gen: 71, Max flush: 111
77572024-02-04T14:03:49.657Z
77582024-02-04T14:03:49.657ZBegin loop 36
77592024-02-04T14:03:49.657ZDownstairs to restart: 0
77602024-02-04T14:03:49.690Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.722337066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417}
77612024-02-04T14:03:49.690Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.722826639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417}
77622024-02-04T14:03:49.693Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
77632024-02-04T14:03:49.693Z,"time":"2024-02-04T14:03:48.72285587Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417}
77642024-02-04T14:03:49.693Z{"msg":"Crucible 8ef43f65-4f7e-4ee7-8af1-2a31d307468d has session id: aceb537c-7af6-4da6-8180-87f178f90126","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723110556Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77652024-02-04T14:03:49.693Z{"msg":"Upstairs opts: Upstairs UUID: 8ef43f65-4f7e-4ee7-8af1-2a31d307468d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.72319619Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77662024-02-04T14:03:49.693Z{"msg":"Crucible stats registered with UUID: 8ef43f65-4f7e-4ee7-8af1-2a31d307468d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723226618Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77672024-02-04T14:03:49.693Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723255448Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417}
77682024-02-04T14:03:49.694Z{"msg":"8ef43f65-4f7e-4ee7-8af1-2a31d307468d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723507726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77692024-02-04T14:03:49.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723583366Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77702024-02-04T14:03:49.694Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723630144Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77712024-02-04T14:03:49.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723712711Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77722024-02-04T14:03:49.694Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723776466Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77732024-02-04T14:03:49.694Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723848149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77742024-02-04T14:03:49.694Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723875309Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77752024-02-04T14:03:49.694Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723904169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"io task","client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77762024-02-04T14:03:49.694Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.723980873Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"io task","client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77772024-02-04T14:03:49.694Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.724148808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"io task","client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77782024-02-04T14:03:49.694Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.724198027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"io task","client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77792024-02-04T14:03:49.694Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.72428787Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"io task","client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77802024-02-04T14:03:49.694Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.724393858Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"io task","client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77812024-02-04T14:03:49.694Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.725215043Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77822024-02-04T14:03:49.694Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.725293423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77832024-02-04T14:03:49.694Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.725340399Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77842024-02-04T14:03:49.694Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.725420835Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77852024-02-04T14:03:49.694Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.726191513Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77862024-02-04T14:03:49.703Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.726585632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77872024-02-04T14:03:49.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.727174948Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77882024-02-04T14:03:49.703Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.727261584Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77892024-02-04T14:03:49.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.727309925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77902024-02-04T14:03:49.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.727435679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77912024-02-04T14:03:49.703Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.727508076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77922024-02-04T14:03:49.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.727535301Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77932024-02-04T14:03:49.703Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.727558475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77942024-02-04T14:03:49.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.727582543Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77952024-02-04T14:03:49.703Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.728362267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77962024-02-04T14:03:49.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729125626Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77972024-02-04T14:03:49.703Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729209653Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77982024-02-04T14:03:49.703Z{"msg":"[0]R flush_numbers[0..12]: [110, 101, 98, 105, 107, 107, 101, 107, 110, 110, 109, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729251651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
77992024-02-04T14:03:49.704Z{"msg":"[0]R generation[0..12]: [70, 64, 62, 67, 68, 68, 64, 68, 70, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.72935156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78002024-02-04T14:03:49.704Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729386459Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78012024-02-04T14:03:49.704Z{"msg":"[1]R flush_numbers[0..12]: [110, 101, 98, 105, 107, 107, 101, 107, 110, 110, 109, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.72941378Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal",Wait for a query_work_queue command to finish before sending IO
78022024-02-04T14:03:49.704Z"pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78032024-02-04T14:03:49.704Z{"msg":"[1]R generation[0..12]: [70, 64, 62, 67, 68, 68, 64, 68, 70, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729530599Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78042024-02-04T14:03:49.704Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729562386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78052024-02-04T14:03:49.704Z{"msg":"[2]R flush_numbers[0..12]: [110, 101, 98, 105, 107, 107, 101, 107, 110, 110, 109, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.72959596Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78062024-02-04T14:03:49.704Z{"msg":"[2]R generation[0..12]: [70, 64, 62, 67, 68, 68, 64, 68, 70, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729665506Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78072024-02-04T14:03:49.704Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729693974Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78082024-02-04T14:03:49.704Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729759173Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78092024-02-04T14:03:49.704Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729831259Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78102024-02-04T14:03:49.704Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729870423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78112024-02-04T14:03:49.704Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.72993165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78122024-02-04T14:03:49.704Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.729999372Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78132024-02-04T14:03:49.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78142024-02-04T14:03:49.704Z,"time":"2024-02-04T14:03:48.730092235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78152024-02-04T14:03:49.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.730202052Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78162024-02-04T14:03:49.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.730231947Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78172024-02-04T14:03:49.704Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.730275275Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78182024-02-04T14:03:49.704Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.730297855Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78192024-02-04T14:03:49.704Z{"msg":"8ef43f65-4f7e-4ee7-8af1-2a31d307468d is now active with session: aceb537c-7af6-4da6-8180-87f178f90126","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.73033833Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78202024-02-04T14:03:49.704Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.73035994Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417,"session_id":"aceb537c-7af6-4da6-8180-87f178f90126"}
78212024-02-04T14:03:49.704Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:48.730399609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417}
78222024-02-04T14:03:49.704ZLoading write count information from file "/tmp/test_repair_verify.data"
78232024-02-04T14:03:49.705ZRun Repair workload
78242024-02-04T14:03:49.708Z01/30 Write block 385 len 36864 data: 6 5 7 7 8 6 9 10 11
78252024-02-04T14:03:49.708Z02/30 Read block 328 len 32768
78262024-02-04T14:03:49.708Z03/30 Read block 208 len 24576
78272024-02-04T14:03:49.708Z04/30 Flush
78282024-02-04T14:03:49.708Z05/30 Write block 249 len 32768 data: 10 12 11 11 7 7 7 6
78292024-02-04T14:03:49.711Z06/30 Write block 177 len 4096 data: 9
78302024-02-04T14:03:49.711Z07/30 Write block 522 len 32768 data: 4 3 3 3 5 4 4 4
78312024-02-04T14:03:49.711Z08/30 Flush
78322024-02-04T14:03:49.715Z09/30 Write block 443 len 32768 data: 5 5 5 5 5 4 4 4
78332024-02-04T14:03:49.718Z10/30 Write block 489 len 24576 data: 9 7 8 10 10 8
78342024-02-04T14:03:49.718Z11/30 Read block 111 len 24576
78352024-02-04T14:03:49.720Z12/30 Flush
78362024-02-04T14:03:49.723Z13/30 Write block 494 len 12288 data: 9 8 8
78372024-02-04T14:03:49.723Z14/30 Read block 329 len 28672
78382024-02-04T14:03:49.726Z15/30 Write block 496 len 24576 data: 9 7 9 10 10 10
78392024-02-04T14:03:49.726Z16/30 Write block 68 len 4096 data: 6
78402024-02-04T14:03:49.730Z17/30 Read block 519 len 36864
78412024-02-04T14:03:49.730Z18/30 Write block 55 len 12288 data: 4 3 4
78422024-02-04T14:03:49.735Z19/30 Read block 322 len 40960
78432024-02-04T14:03:49.735Z20/30 Flush
78442024-02-04T14:03:49.738Z21/30 Write block 251 len 40960 data: 12 12 8 8 8 7 6 7 7 5
78452024-02-04T14:03:49.738Z22/30 Read block 43 len 24576
78462024-02-04T14:03:49.741Z23/30 Read block 490 len 40960
78472024-02-04T14:03:49.741Z24/30 Read block 222 len 28672
78482024-02-04T14:03:49.744Z25/30 Write block 54 len 40960 data: 4 5 4 5 4 6 8 7 7 5
78492024-02-04T14:03:49.744Z26/30 Write block 369 len 12288 data: 6 6 4
78502024-02-04T14:03:49.747Z27/30 Write block 171 len 8192 data: 7 11
78512024-02-04T14:03:49.747Z28/30 Read block 219 len 20480
78522024-02-04T14:03:49.750Z29/30 Read block 238 len 16384
78532024-02-04T14:03:49.750Z30/30 Write block 444 len 40960 data: 6 6 6 6 5 5 5 5 6 5
78542024-02-04T14:03:49.753Z----------------------------------------------------------------
78552024-02-04T14:03:49.753Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78562024-02-04T14:03:49.753ZStates: Active Active Active
78572024-02-04T14:03:49.753ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78582024-02-04T14:03:49.753Z 1 Acked 1000 Write 9 Sent Done Done false
78592024-02-04T14:03:49.753Z 2 Acked 1001 Read 8 Sent Done Done false
78602024-02-04T14:03:49.753Z 3 Acked 1002 Read 6 Sent Done Done false
78612024-02-04T14:03:49.753Z 4 Acked 1003 Flush 0 Sent Done Done false
78622024-02-04T14:03:49.753Z 5 Acked 1004 Write 8 Sent Done Done false
78632024-02-04T14:03:49.753Z 6 Acked 1005 Write 1 Sent Done Done false
78642024-02-04T14:03:49.753Z 7 Acked 1006 Write 8 Sent Done Done false
78652024-02-04T14:03:49.753Z 8 Acked 1007 Flush 0 Sent Done Done false
78662024-02-04T14:03:49.754Z 9 Acked 1008 Write 8 Sent Done Done false
78672024-02-04T14:03:49.754Z 10 Acked 1009 Write 6 Sent Done Done false
78682024-02-04T14:03:49.754Z 11 Acked 1010 Read 6 Sent Done Done false
78692024-02-04T14:03:49.754Z 12 Acked 1011 Flush 0 Sent Done Done false
78702024-02-04T14:03:49.754Z 13 Acked 1012 Write 3 Sent Done Done false
78712024-02-04T14:03:49.754Z 14 Acked 1013 Read 7 Sent Done Done false
78722024-02-04T14:03:49.754Z 15 Acked 1014 Write 6 Sent Done Done false
78732024-02-04T14:03:49.754Z 16 Acked 1015 Write 1 Sent Done Done false
78742024-02-04T14:03:49.754Z 17 Acked 1016 Read 9 Sent Done Done false
78752024-02-04T14:03:49.754Z 18 Acked 1017 Write 3 Sent Done Done false
78762024-02-04T14:03:49.754Z 19 Acked 1018 Read 10 Sent Done Done false
78772024-02-04T14:03:49.754Z 20 Acked 1019 Flush 0 Sent Done Done false
78782024-02-04T14:03:49.754Z 21 Acked 1020 Write 10 Sent Done Done false
78792024-02-04T14:03:49.754Z 22 Acked 1021 Read 6 Sent Done Done false
78802024-02-04T14:03:49.754Z 23 Acked 1022 Read 10 Sent Done Done false
78812024-02-04T14:03:49.754Z 24 Acked 1023 Read 7 Sent Done Done false
78822024-02-04T14:03:49.754Z 25 Acked 1024 Write 10 Sent Done Done false
78832024-02-04T14:03:49.754Z 26 Acked 1025 Write 3 Sent Done Done false
78842024-02-04T14:03:49.754Z 27 Acked 1026 Write 2 Sent Done Done false
78852024-02-04T14:03:49.754Z 28 Acked 1027 Read 5 Sent Done Done false
78862024-02-04T14:03:49.754Z 29 Acked 1028 Read 4 Sent Done Done false
78872024-02-04T14:03:49.754Z 30 Acked 1029 Write 10 Sent Sent Sent false
78882024-02-04T14:03:49.754Z STATES DS:0 DS:1 DS:2 TOTAL
78892024-02-04T14:03:49.754Z New 0 0 0 0
78902024-02-04T14:03:49.754Z Sent 30 1 1 32
78912024-02-04T14:03:49.755Z Done 0 29 29 58
78922024-02-04T14:03:49.755Z Skipped 0 0 0 0
78932024-02-04T14:03:49.755Z Error 0 0 0 0
78942024-02-04T14:03:49.755ZLast Flush: 0 1019 1019
78952024-02-04T14:03:49.755ZDownstairs last five completed:
78962024-02-04T14:03:49.755ZUpstairs last five completed: 30 29 28 27 26
78972024-02-04T14:03:49.755Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:48.784190175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1417}
78982024-02-04T14:03:49.755ZWrote out file "/tmp/test_repair_verify.data"
78992024-02-04T14:03:49.759Z
79002024-02-04T14:03:49.765ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
79012024-02-04T14:03:49.794ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79022024-02-04T14:03:49.794Z 0 000-019 70 70 70 110 110 110 F F F
79032024-02-04T14:03:49.794Z 1 020-039 64 64 64 101 101 101 F F F
79042024-02-04T14:03:49.794Z 2 040-059 62 72 72 98 115 115 F T T <---
79052024-02-04T14:03:49.794Z 3 060-079 67 72 72 105 115 115 F T T <---
79062024-02-04T14:03:49.794Z 4 080-099 68 68 68 107 107 107 F F F
79072024-02-04T14:03:49.794Z 5 100-119 68 68 68 107 107 107 F F F
79082024-02-04T14:03:49.794Z 6 120-139 64 64 64 101 101 101 F F F
79092024-02-04T14:03:49.794Z 7 140-159 68 68 68 107 107 107 F F F
79102024-02-04T14:03:49.794Z 8 160-179 70 72 72 110 113 113 F T T <---
79112024-02-04T14:03:49.794Z 9 180-199 70 70 70 110 110 110 F F F
79122024-02-04T14:03:49.794Z 10 200-219 70 70 70 109 109 109 F F F
79132024-02-04T14:03:49.794Z 11 220-239 68 68 68 107 107 107 F F F
79142024-02-04T14:03:49.795Z 12 240-259 70 72 72 109 113 113 F T T <---
79152024-02-04T14:03:49.795Z 13 260-279 70 70 70 109 109 109 F T T <---
79162024-02-04T14:03:49.795Z 14 280-299 70 70 70 110 110 110 F F F
79172024-02-04T14:03:49.795Z 15 300-319 66 66 66 104 104 104 F F F
79182024-02-04T14:03:49.798Z 16 320-339 64 64 64 101 101 101 F F F
79192024-02-04T14:03:49.798Z 17 340-359 71 71 71 111 111 111 F F F
79202024-02-04T14:03:49.798Z 18 360-379 71 71 71 111 111 111 F T T <---
79212024-02-04T14:03:49.798Z 19 380-399 68 72 72 107 112 112 F F F <---
79222024-02-04T14:03:49.798Z 20 400-419 69 69 69 108 108 108 F F F
79232024-02-04T14:03:49.798Z 21 420-439 67 67 67 105 105 105 F F F
79242024-02-04T14:03:49.798Z 22 440-459 70 72 72 110 114 114 F T T <---
79252024-02-04T14:03:49.798Z 23 460-479 66 66 66 103 103 103 F F F
79262024-02-04T14:03:49.798Z 24 480-499 71 72 72 111 115 115 F F F <---
79272024-02-04T14:03:49.798Z 25 500-519 70 72 72 109 115 115 F F F <---
79282024-02-04T14:03:49.798Z 26 520-539 65 72 72 102 113 113 F F F <---
79292024-02-04T14:03:49.798Z 27 540-559 64 64 64 101 101 101 F F F
79302024-02-04T14:03:49.798Z 28 560-579 70 70 70 110 110 110 F F F
79312024-02-04T14:03:49.798Z 29 580-599 70 70 70 109 109 109 F F F
79322024-02-04T14:03:49.798ZMax gen: 72, Max flush: 115
79332024-02-04T14:03:49.798ZError: Difference in extent metadata found!
79342024-02-04T14:03:49.801ZA Difference in extent metadata is expected here
79352024-02-04T14:03:49.801ZOn loop 36
79362024-02-04T14:03:49.801Z
79372024-02-04T14:03:49.804ZVerifying data now
79382024-02-04T14:03:54.831ZNo change after verify
79392024-02-04T14:03:54.831ZLoop: 36 Downstairs dump after verify (and repair):
79402024-02-04T14:03:54.854ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79412024-02-04T14:03:54.854Z 0 000-019 70 70 70 110 110 110 F F F
79422024-02-04T14:03:54.854Z 1 020-039 64 64 64 101 101 101 F F F
79432024-02-04T14:03:54.855Z 2 040-059 73 73 73 116 116 116 F F F
79442024-02-04T14:03:54.855Z 3 060-079 73 73 73 116 116 116 F F F
79452024-02-04T14:03:54.855Z 4 080-099 68 68 68 107 107 107 F F F
79462024-02-04T14:03:54.855Z 5 100-119 68 68 68 107 107 107 F F F
79472024-02-04T14:03:54.855Z 6 120-139 64 64 64 101 101 101 F F F
79482024-02-04T14:03:54.855Z 7 140-159 68 68 68 107 107 107 F F F
79492024-02-04T14:03:54.855Z 8 160-179 73 73 73 116 116 116 F F F
79502024-02-04T14:03:54.855Z 9 180-199 70 70 70 110 110 110 F F F
79512024-02-04T14:03:54.855Z 10 200-219 70 70 70 109 109 109 F F F
79522024-02-04T14:03:54.860Z 11 220-239 68 68 68 107 107 107 F F F
79532024-02-04T14:03:54.860Z 12 240-259 73 73 73 116 116 116 F F F
79542024-02-04T14:03:54.860Z 13 260-279 73 73 73 116 116 116 F F F
79552024-02-04T14:03:54.860Z 14 280-299 70 70 70 110 110 110 F F F
79562024-02-04T14:03:54.860Z 15 300-319 66 66 66 104 104 104 F F F
79572024-02-04T14:03:54.860Z 16 320-339 64 64 64 101 101 101 F F F
79582024-02-04T14:03:54.860Z 17 340-359 71 71 71 111 111 111 F F F
79592024-02-04T14:03:54.861Z 18 360-379 73 73 73 116 116 116 F F F
79602024-02-04T14:03:54.861Z 19 380-399 72 72 72 112 112 112 F F F
79612024-02-04T14:03:54.861Z 20 400-419 69 69 69 108 108 108 F F F
79622024-02-04T14:03:54.861Z 21 420-439 67 67 67 105 105 105 F F F
79632024-02-04T14:03:54.861Z 22 440-459 73 73 73 116 116 116 F F F
79642024-02-04T14:03:54.861Z 23 460-479 66 66 66 103 103 103 F F F
79652024-02-04T14:03:54.861Z 24 480-499 72 72 72 115 115 115 F F F
79662024-02-04T14:03:54.861Z 25 500-519 72 72 72 115 115 115 F F F
79672024-02-04T14:03:54.861Z 26 520-539 72 72 72 113 113 113 F F F
79682024-02-04T14:03:54.861Z 27 540-559 64 64 64 101 101 101 F F F
79692024-02-04T14:03:54.861Z 28 560-579 70 70 70 110 110 110 F F F
79702024-02-04T14:03:54.861Z 29 580-599 70 70 70 109 109 109 F F F
79712024-02-04T14:03:54.861ZMax gen: 73, Max flush: 116
79722024-02-04T14:03:54.865Z
79732024-02-04T14:03:54.865ZBegin loop 37
79742024-02-04T14:03:54.865ZDownstairs to restart: 1
79752024-02-04T14:03:54.892Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.925135314Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425}
79762024-02-04T14:03:54.892Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.925593064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425}
79772024-02-04T14:03:54.892Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.925622314Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425}
79782024-02-04T14:03:54.892Z{"msg":"Crucible adbfbd9b-2dea-40d0-a2e9-2084239c8705 has session id: b7b7a41d-fa7a-488c-bdd4-ce061392747b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.925645941Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79792024-02-04T14:03:54.896Z{"msg":"Upstairs opts: Upstairs UUID: adbfbd9b-2dea-40d0-a2e9-2084239c8705, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.925731102Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79802024-02-04T14:03:54.896Z{"msg":"Crucible stats registered with UUID: adbfbd9b-2dea-40d0-a2e9-2084239c8705","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.925819789Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"Crucible runtime is spawned
79812024-02-04T14:03:54.897Zb7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79822024-02-04T14:03:54.897Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926034015Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425}
79832024-02-04T14:03:54.897Z{"msg":"adbfbd9b-2dea-40d0-a2e9-2084239c8705 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926197031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79842024-02-04T14:03:54.897Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926282082Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79852024-02-04T14:03:54.897Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926395071Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79862024-02-04T14:03:54.897Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926466783Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79872024-02-04T14:03:54.897Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926511573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79882024-02-04T14:03:54.897Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926552955Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79892024-02-04T14:03:54.897Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926594084Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79902024-02-04T14:03:54.897Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926669383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"io task","client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79912024-02-04T14:03:54.897Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926755222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"io task","client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79922024-02-04T14:03:54.897Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.926821237Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"io task","client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79932024-02-04T14:03:54.897Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.927003871Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"io task","client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79942024-02-04T14:03:54.897Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.927126786Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"io task","client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79952024-02-04T14:03:54.897Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.927173465Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"io task","client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79962024-02-04T14:03:54.898Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.927868925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79972024-02-04T14:03:54.898Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.927931723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79982024-02-04T14:03:54.898Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.928122831Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
79992024-02-04T14:03:54.898Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.92820334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80002024-02-04T14:03:54.898Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.928746281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80012024-02-04T14:03:54.898Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.929093292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80022024-02-04T14:03:54.902Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.929722122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80032024-02-04T14:03:54.903Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.929796813Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80042024-02-04T14:03:54.903Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.929875475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80052024-02-04T14:03:54.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.929942983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80062024-02-04T14:03:54.903Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.929971006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80072024-02-04T14:03:54.903Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.930026321Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80082024-02-04T14:03:54.903Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.930185331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80092024-02-04T14:03:54.903Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.930252894Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80102024-02-04T14:03:54.903Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.931008418Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80112024-02-04T14:03:54.903Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.931703811Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80122024-02-04T14:03:54.903Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.93177567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80132024-02-04T14:03:54.903Z{"msg":"[0]R flush_numbers[0..12]: [110, 101, 116, 116, 107, 107, 101, 107, 116, 110, 109, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.931802091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80142024-02-04T14:03:54.903Z{"msg":"[0]R generation[0..12]: [70, 64, 73, 73, 68, 68, 64, 68, 73, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.931861166Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80152024-02-04T14:03:54.903Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.931894661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80162024-02-04T14:03:54.903Z{"msg":"[1]R flush_numbers[0..12]: [110, 101, 116, 116, 107, 107, 101, 107, 116, 110, 109, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.931955786Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80172024-02-04T14:03:54.903Z{"msg":"[1]R generation[0..12]: [70, 64, 73, 73, 68, 68, 64, 68, 73, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.931982527Z","Wait for a query_work_queue command to finish before sending IO
80182024-02-04T14:03:54.903Zhostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80192024-02-04T14:03:54.903Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932076659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80202024-02-04T14:03:54.903Z{"msg":"[2]R flush_numbers[0..12]: [110, 101, 116, 116, 107, 107, 101, 107, 116, 110, 109, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932104761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80212024-02-04T14:03:54.903Z{"msg":"[2]R generation[0..12]: [70, 64, 73, 73, 68, 68, 64, 68, 73, 70, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932162468Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80222024-02-04T14:03:54.903Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932190554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80232024-02-04T14:03:54.903Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932254538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80242024-02-04T14:03:54.903Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932280961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80252024-02-04T14:03:54.903Z{"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932341076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80262024-02-04T14:03:54.904Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932367003Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80272024-02-04T14:03:54.904Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932423669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80282024-02-04T14:03:54.904Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932447866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80292024-02-04T14:03:54.904Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932477343Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80302024-02-04T14:03:54.904Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932531245Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80312024-02-04T14:03:54.904Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932556161Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80322024-02-04T14:03:54.904Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932619194Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80332024-02-04T14:03:54.904Z{"msg":"adbfbd9b-2dea-40d0-a2e9-2084239c8705 is now active with session: b7b7a41d-fa7a-488c-bdd4-ce061392747b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932645038Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80342024-02-04T14:03:54.904Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.93267935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425,"session_id":"b7b7a41d-fa7a-488c-bdd4-ce061392747b"}
80352024-02-04T14:03:54.904ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80362024-02-04T14:03:54.904Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:53.932751485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425}
80372024-02-04T14:03:54.904ZLoading write count information from file "/tmp/test_repair_verify.data"
80382024-02-04T14:03:54.904ZRun Repair workload
80392024-02-04T14:03:54.904Z01/30 Write block 355 len 32768 data: 9 10 6 5 5 5 7 7
80402024-02-04T14:03:54.907Z02/30 Write block 225 len 40960 data: 5 5 6 6 5 4 3 5 6 6
80412024-02-04T14:03:54.907Z03/30 Read block 373 len 24576
80422024-02-04T14:03:54.908Z04/30 Read block 0 len 28672
80432024-02-04T14:03:54.911Z05/30 Read block 263 len 20480
80442024-02-04T14:03:54.911Z06/30 Read block 199 len 40960
80452024-02-04T14:03:54.914Z07/30 Write block 23 len 20480 data: 6 5 5 5 5
80462024-02-04T14:03:54.914Z08/30 Write block 103 len 12288 data: 3 5 5
80472024-02-04T14:03:54.914Z09/30 Read block 49 len 12288
80482024-02-04T14:03:54.919Z10/30 Write block 502 len 32768 data: 7 7 8 8 10 10 9 9
80492024-02-04T14:03:54.919Z11/30 Read block 123 len 28672
80502024-02-04T14:03:54.923Z12/30 Read block 350 len 40960
80512024-02-04T14:03:54.923Z13/30 Write block 559 len 20480 data: 7 8 8 7 7
80522024-02-04T14:03:54.923Z14/30 Flush
80532024-02-04T14:03:54.927Z15/30 Write block 430 len 4096 data: 6
80542024-02-04T14:03:54.927Z16/30 Read block 566 len 32768
80552024-02-04T14:03:54.930Z17/30 Write block 548 len 12288 data: 3 4 5
80562024-02-04T14:03:54.930Z18/30 Read block 541 len 40960
80572024-02-04T14:03:54.933Z19/30 Read block 86 len 32768
80582024-02-04T14:03:54.933Z20/30 Flush
80592024-02-04T14:03:54.936Z21/30 Write block 304 len 16384 data: 6 7 6 7
80602024-02-04T14:03:54.937Z22/30 Write block 257 len 4096 data: 7
80612024-02-04T14:03:54.951Z23/30 Read block 443 len 32768
80622024-02-04T14:03:54.951Z24/30 Flush
80632024-02-04T14:03:54.951Z25/30 Write block 43 len 12288 data: 5 6 6
80642024-02-04T14:03:54.951Z26/30 Read block 367 len 16384
80652024-02-04T14:03:54.955Z27/30 Write block 418 len 4096 data: 5
80662024-02-04T14:03:54.955Z28/30 Read block 22 len 4096
80672024-02-04T14:03:54.955Z29/30 Read block 200 len 20480
80682024-02-04T14:03:54.955Z30/30 Read block 544 len 12288
80692024-02-04T14:03:54.955Z----------------------------------------------------------------
80702024-02-04T14:03:54.955Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:16
80712024-02-04T14:03:54.955ZStates: Active Active Active
80722024-02-04T14:03:54.955ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80732024-02-04T14:03:54.955Z 15 Acked 1014 Write 1 Done Done Done false
80742024-02-04T14:03:54.955Z 16 Acked 1015 Read 8 Done Done Done false
80752024-02-04T14:03:54.955Z 17 Acked 1016 Write 3 Done Done Done false
80762024-02-04T14:03:54.955Z 18 Acked 1017 Read 10 Done Sent Done false
80772024-02-04T14:03:54.955Z 19 Acked 1018 Read 8 Done Sent Done false
80782024-02-04T14:03:54.955Z 20 Acked 1019 Flush 0 Done Sent Done false
80792024-02-04T14:03:54.955Z 21 Acked 1020 Write 4 Done Sent Done false
80802024-02-04T14:03:54.955Z 22 Acked 1021 Write 1 Done Sent Done false
80812024-02-04T14:03:54.956Z 23 Acked 1022 Read 8 Done Sent Done false
80822024-02-04T14:03:54.956Z 24 Acked 1023 Flush 0 Done Sent Done false
80832024-02-04T14:03:54.956Z 25 Acked 1024 Write 3 Done Sent Done false
80842024-02-04T14:03:54.956Z 26 Acked 1025 Read 4 Done Sent Done false
80852024-02-04T14:03:54.956Z 27 Acked 1026 Write 1 Done Sent Done false
80862024-02-04T14:03:54.956Z 28 Acked 1027 Read 1 Done Sent Done false
80872024-02-04T14:03:54.956Z 29 Acked 1028 Read 5 Done Sent Done false
80882024-02-04T14:03:54.956Z 30 Acked 1029 Read 3 Sent Sent Done false
80892024-02-04T14:03:54.956Z STATES DS:0 DS:1 DS:2 TOTAL
80902024-02-04T14:03:54.956Z New 0 0 0 0
80912024-02-04T14:03:54.956Z Sent 1 13 0 14
80922024-02-04T14:03:54.956Z Done 15 3 16 34
80932024-02-04T14:03:54.956Z Skipped 0 0 0 0
80942024-02-04T14:03:54.956Z Error 0 0 0 0
80952024-02-04T14:03:54.956ZLast Flush: 1023 1013 1023
80962024-02-04T14:03:54.956ZDownstairs last five completed: 1013 1012 1011 1010 1009
80972024-02-04T14:03:54.956ZUpstairs last five completed: 30 29 28 27 26
80982024-02-04T14:03:54.956Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:53.981010943Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1425}
80992024-02-04T14:03:54.956ZWrote out file "/tmp/test_repair_verify.data"
81002024-02-04T14:03:54.960Z
81012024-02-04T14:03:54.963ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81022024-02-04T14:03:54.987ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81032024-02-04T14:03:54.987Z 0 000-019 70 70 70 110 110 110 F F F
81042024-02-04T14:03:54.987Z 1 020-039 74 74 74 117 117 117 F F F
81052024-02-04T14:03:54.987Z 2 040-059 73 73 73 116 116 116 T F T <---
81062024-02-04T14:03:54.987Z 3 060-079 73 73 73 116 116 116 F F F
81072024-02-04T14:03:54.987Z 4 080-099 68 68 68 107 107 107 F F F
81082024-02-04T14:03:54.987Z 5 100-119 74 74 74 117 117 117 F F F
81092024-02-04T14:03:54.987Z 6 120-139 64 64 64 101 101 101 F F F
81102024-02-04T14:03:54.987Z 7 140-159 68 68 68 107 107 107 F F F
81112024-02-04T14:03:54.987Z 8 160-179 73 73 73 116 116 116 F F F
81122024-02-04T14:03:54.987Z 9 180-199 70 70 70 110 110 110 F F F
81132024-02-04T14:03:54.987Z 10 200-219 70 70 70 109 109 109 F F F
81142024-02-04T14:03:54.988Z 11 220-239 74 74 74 117 117 117 F F F
81152024-02-04T14:03:54.991Z 12 240-259 74 73 74 119 116 119 F F F <---
81162024-02-04T14:03:54.991Z 13 260-279 73 73 73 116 116 116 F F F
81172024-02-04T14:03:54.991Z 14 280-299 70 70 70 110 110 110 F F F
81182024-02-04T14:03:54.991Z 15 300-319 74 66 74 119 104 119 F F F <---
81192024-02-04T14:03:54.991Z 16 320-339 64 64 64 101 101 101 F F F
81202024-02-04T14:03:54.991Z 17 340-359 74 74 74 117 117 117 F F F
81212024-02-04T14:03:54.991Z 18 360-379 74 74 74 117 117 117 F F F
81222024-02-04T14:03:54.992Z 19 380-399 72 72 72 112 112 112 F F F
81232024-02-04T14:03:54.992Z 20 400-419 69 69 69 108 108 108 T F T <---
81242024-02-04T14:03:54.992Z 21 420-439 74 67 74 118 105 118 F T F <---
81252024-02-04T14:03:54.992Z 22 440-459 73 73 73 116 116 116 F F F
81262024-02-04T14:03:54.992Z 23 460-479 66 66 66 103 103 103 F F F
81272024-02-04T14:03:54.992Z 24 480-499 72 72 72 115 115 115 F F F
81282024-02-04T14:03:54.992Z 25 500-519 74 74 74 117 117 117 F F F
81292024-02-04T14:03:54.992Z 26 520-539 72 72 72 113 113 113 F F F
81302024-02-04T14:03:54.992Z 27 540-559 74 74 74 118 117 118 F T F <---
81312024-02-04T14:03:54.992Z 28 560-579 74 74 74 117 117 117 F F F
81322024-02-04T14:03:54.992Z 29 580-599 70 70 70 109 109 109 F F F
81332024-02-04T14:03:54.992ZMax gen: 74, Max flush: 119
81342024-02-04T14:03:54.993ZError: Difference in extent metadata found!
81352024-02-04T14:03:54.996ZA Difference in extent metadata is expected here
81362024-02-04T14:03:54.996ZOn loop 37
81372024-02-04T14:03:54.996Z
81382024-02-04T14:03:54.996ZVerifying data now
81392024-02-04T14:03:59.616ZNo change after verify
81402024-02-04T14:03:59.616ZLoop: 37 Downstairs dump after verify (and repair):
81412024-02-04T14:03:59.639ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81422024-02-04T14:03:59.639Z 0 000-019 70 70 70 110 110 110 F F F
81432024-02-04T14:03:59.639Z 1 020-039 74 74 74 117 117 117 F F F
81442024-02-04T14:03:59.639Z 2 040-059 75 75 75 120 120 120 F F F
81452024-02-04T14:03:59.639Z 3 060-079 73 73 73 116 116 116 F F F
81462024-02-04T14:03:59.639Z 4 080-099 68 68 68 107 107 107 F F F
81472024-02-04T14:03:59.639Z 5 100-119 74 74 74 117 117 117 F F F
81482024-02-04T14:03:59.642Z 6 120-139 64 64 64 101 101 101 F F F
81492024-02-04T14:03:59.642Z 7 140-159 68 68 68 107 107 107 F F F
81502024-02-04T14:03:59.642Z 8 160-179 73 73 73 116 116 116 F F F
81512024-02-04T14:03:59.642Z 9 180-199 70 70 70 110 110 110 F F F
81522024-02-04T14:03:59.642Z 10 200-219 70 70 70 109 109 109 F F F
81532024-02-04T14:03:59.642Z 11 220-239 74 74 74 117 117 117 F F F
81542024-02-04T14:03:59.642Z 12 240-259 74 74 74 119 119 119 F F F
81552024-02-04T14:03:59.642Z 13 260-279 73 73 73 116 116 116 F F F
81562024-02-04T14:03:59.642Z 14 280-299 70 70 70 110 110 110 F F F
81572024-02-04T14:03:59.642Z 15 300-319 74 74 74 119 119 119 F F F
81582024-02-04T14:03:59.642Z 16 320-339 64 64 64 101 101 101 F F F
81592024-02-04T14:03:59.643Z 17 340-359 74 74 74 117 117 117 F F F
81602024-02-04T14:03:59.643Z 18 360-379 74 74 74 117 117 117 F F F
81612024-02-04T14:03:59.643Z 19 380-399 72 72 72 112 112 112 F F F
81622024-02-04T14:03:59.643Z 20 400-419 75 75 75 120 120 120 F F F
81632024-02-04T14:03:59.643Z 21 420-439 74 74 74 118 118 118 F F F
81642024-02-04T14:03:59.643Z 22 440-459 73 73 73 116 116 116 F F F
81652024-02-04T14:03:59.643Z 23 460-479 66 66 66 103 103 103 F F F
81662024-02-04T14:03:59.643Z 24 480-499 72 72 72 115 115 115 F F F
81672024-02-04T14:03:59.643Z 25 500-519 74 74 74 117 117 117 F F F
81682024-02-04T14:03:59.643Z 26 520-539 72 72 72 113 113 113 F F F
81692024-02-04T14:03:59.643Z 27 540-559 74 74 74 118 118 118 F F F
81702024-02-04T14:03:59.643Z 28 560-579 74 74 74 117 117 117 F F F
81712024-02-04T14:03:59.643Z 29 580-599 70 70 70 109 109 109 F F F
81722024-02-04T14:03:59.643ZMax gen: 75, Max flush: 120
81732024-02-04T14:03:59.646Z
81742024-02-04T14:03:59.646ZBegin loop 38
81752024-02-04T14:03:59.646ZDownstairs to restart: 2
81762024-02-04T14:03:59.674Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.707563658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433}
81772024-02-04T14:03:59.678Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708009389Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433}
81782024-02-04T14:03:59.678Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708100354Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433}
81792024-02-04T14:03:59.678Z{"msg":"Crucible 43de9ed3-9257-4099-82f7-da2affd2575e has session id: f1e2a99f-bb7c-4b2f-8baf-d72e416a6133","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708140752Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81802024-02-04T14:03:59.678Z{"msg":"Crucible runtime is spawned
81812024-02-04T14:03:59.678ZUpstairs opts: Upstairs UUID: 43de9ed3-9257-4099-82f7-da2affd2575e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708194898Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81822024-02-04T14:03:59.678Z{"msg":"Crucible stats registered with UUID: 43de9ed3-9257-4099-82f7-da2affd2575e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708299589Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81832024-02-04T14:03:59.678Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708341292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433}
81842024-02-04T14:03:59.678Z{"msg":"43de9ed3-9257-4099-82f7-da2affd2575e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708516556Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81852024-02-04T14:03:59.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708576432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81862024-02-04T14:03:59.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708620874Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81872024-02-04T14:03:59.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708690692Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81882024-02-04T14:03:59.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708729746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81892024-02-04T14:03:59.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708753468Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81902024-02-04T14:03:59.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708853951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81912024-02-04T14:03:59.679Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708898474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81922024-02-04T14:03:59.679Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.708974038Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81932024-02-04T14:03:59.679Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.709093344Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81942024-02-04T14:03:59.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.709138251Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81952024-02-04T14:03:59.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.709219083Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81962024-02-04T14:03:59.679Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.709262076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81972024-02-04T14:03:59.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.710345832Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81982024-02-04T14:03:59.679Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.710412733Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
81992024-02-04T14:03:59.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.71045319Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82002024-02-04T14:03:59.679Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.710545474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82012024-02-04T14:03:59.679Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.710592472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82022024-02-04T14:03:59.679Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.710673013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82032024-02-04T14:03:59.683Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.711500536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82042024-02-04T14:03:59.683Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.711714236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82052024-02-04T14:03:59.683Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.711857719Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82062024-02-04T14:03:59.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.712640781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82072024-02-04T14:03:59.683Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.712705259Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82082024-02-04T14:03:59.683Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.712731488Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82092024-02-04T14:03:59.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713057242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82102024-02-04T14:03:59.683Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713121184Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82112024-02-04T14:03:59.683Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713147248Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82122024-02-04T14:03:59.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713186688Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82132024-02-04T14:03:59.684Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713210293Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82142024-02-04T14:03:59.684Z{"msg":"[0]R flush_numbers[0..12]: [110, 117, 120, 116, 107, 117, 101, 107, 116, 110, 109, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713252077Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82152024-02-04T14:03:59.684Z{"msg":"[0]R generation[0..12]: [70, 74, 75, 73, 68, 74, 64, 68, 73, 70, 70, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713276424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82162024-02-04T14:03:59.684Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713300558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82172024-02-04T14:03:59.684Z{"msg":"[1]R flush_numbers[0..12]: [110, 117, 120, 116, 107, 117, 101, 107, 116, 110, 109, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713324146Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82182024-02-04T14:03:59.684Z{"msg":"[1]R generation[0..12]: [70, 74, 75, 73, 68, 74, 64, 68, 73, 70, 70, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713357107Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82192024-02-04T14:03:59.684Z{"msg":"Wait for a query_work_queue command to finish before sending IO
82202024-02-04T14:03:59.684Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713495398Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82212024-02-04T14:03:59.684Z{"msg":"[2]R flush_numbers[0..12]: [110, 117, 120, 116, 107, 117, 101, 107, 116, 110, 109, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.71357269Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82222024-02-04T14:03:59.684Z{"msg":"[2]R generation[0..12]: [70, 74, 75, 73, 68, 74, 64, 68, 73, 70, 70, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.71360197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82232024-02-04T14:03:59.684Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713626982Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82242024-02-04T14:03:59.684Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713650268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82252024-02-04T14:03:59.684Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713672366Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82262024-02-04T14:03:59.684Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713693515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82272024-02-04T14:03:59.684Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713715308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82282024-02-04T14:03:59.684Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713738695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82292024-02-04T14:03:59.684Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713778345Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82302024-02-04T14:03:59.684Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713801662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82312024-02-04T14:03:59.684Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.71382353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82322024-02-04T14:03:59.684Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713845701Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82332024-02-04T14:03:59.684Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713866107Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82342024-02-04T14:03:59.684Z{"msg":"43de9ed3-9257-4099-82f7-da2affd2575e is now active with session: f1e2a99f-bb7c-4b2f-8baf-d72e416a6133","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713906658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82352024-02-04T14:03:59.684Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713927799Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433,"session_id":"f1e2a99f-bb7c-4b2f-8baf-d72e416a6133"}
82362024-02-04T14:03:59.684Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:03:58.713948478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433}
82372024-02-04T14:03:59.684ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82382024-02-04T14:03:59.685ZLoading write count information from file "/tmp/test_repair_verify.data"
82392024-02-04T14:03:59.685ZRun Repair workload
82402024-02-04T14:03:59.685Z01/30 Write block 347 len 28672 data: 6 5 7 5 7 9 10
82412024-02-04T14:03:59.688Z02/30 Read block 316 len 20480
82422024-02-04T14:03:59.688Z03/30 Read block 164 len 16384
82432024-02-04T14:03:59.691Z04/30 Read block 447 len 20480
82442024-02-04T14:03:59.691Z05/30 Write block 157 len 16384 data: 4 3 5 4
82452024-02-04T14:03:59.693Z06/30 Write block 506 len 24576 data: 11 11 10 10 8 7
82462024-02-04T14:03:59.693Z07/30 Read block 437 len 28672
82472024-02-04T14:03:59.696Z08/30 Flush
82482024-02-04T14:03:59.699Z09/30 Write block 417 len 20480 data: 7 6 8 11 10
82492024-02-04T14:03:59.699Z10/30 Read block 309 len 20480
82502024-02-04T14:03:59.704Z11/30 Write block 450 len 28672 data: 6 6 7 6 8 7 8
82512024-02-04T14:03:59.704Z12/30 Read block 586 len 16384
82522024-02-04T14:03:59.707Z13/30 Write block 395 len 12288 data: 8 8 10
82532024-02-04T14:03:59.707Z14/30 Write block 497 len 32768 data: 8 10 11 11 11 8 8 9
82542024-02-04T14:03:59.707Z15/30 Read block 360 len 28672
82552024-02-04T14:03:59.707Z16/30 Read block 471 len 4096
82562024-02-04T14:03:59.710Z17/30 Read block 294 len 32768
82572024-02-04T14:03:59.710Z18/30 Read block 20 len 20480
82582024-02-04T14:03:59.713Z19/30 Read block 581 len 12288
82592024-02-04T14:03:59.713Z20/30 Write block 553 len 32768 data: 5 5 5 7 8 7 8 9
82602024-02-04T14:03:59.716Z21/30 Read block 243 len 20480
82612024-02-04T14:03:59.716Z22/30 Read block 219 len 32768
82622024-02-04T14:03:59.719Z23/30 Write block 428 len 36864 data: 5 5 7 6 6 6 7 7 6
82632024-02-04T14:03:59.719Z24/30 Write block 367 len 20480 data: 7 6 7 7 5
82642024-02-04T14:03:59.722Z25/30 Write block 500 len 20480 data: 12 12 9 9 10
82652024-02-04T14:03:59.722Z26/30 Read block 377 len 20480
82662024-02-04T14:03:59.724Z27/30 Read block 65 len 20480
82672024-02-04T14:03:59.724Z28/30 Write block 524 len 4096 data: 4
82682024-02-04T14:03:59.728Z29/30 Write block 204 len 16384 data: 11 8 7 6
82692024-02-04T14:03:59.728Z30/30 Write block 371 len 20480 data: 6 4 4 7 8
82702024-02-04T14:03:59.728Z----------------------------------------------------------------
82712024-02-04T14:03:59.728Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
82722024-02-04T14:03:59.728ZStates: Active Active Active
82732024-02-04T14:03:59.728ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82742024-02-04T14:03:59.728Z 1 Acked 1000 Write 7 Done Done Done false
82752024-02-04T14:03:59.728Z 2 Acked 1001 Read 5 Done Done Done false
82762024-02-04T14:03:59.728Z 3 Acked 1002 Read 4 Done Done Done false
82772024-02-04T14:03:59.728Z 4 Acked 1003 Read 5 Done Done Done false
82782024-02-04T14:03:59.728Z 5 Acked 1004 Write 4 Done Done Sent false
82792024-02-04T14:03:59.728Z 6 Acked 1005 Write 6 Done Done Sent false
82802024-02-04T14:03:59.728Z 7 Acked 1006 Read 7 Done Done Sent false
82812024-02-04T14:03:59.728Z 8 Acked 1007 Flush 0 Done Done Sent false
82822024-02-04T14:03:59.729Z 9 Acked 1008 Write 5 Done Done Sent false
82832024-02-04T14:03:59.729Z 10 Acked 1009 Read 5 Done Done Sent false
82842024-02-04T14:03:59.729Z 11 Acked 1010 Write 7 Done Done Sent false
82852024-02-04T14:03:59.729Z 12 Acked 1011 Read 4 Done Done Sent false
82862024-02-04T14:03:59.729Z 13 Acked 1012 Write 3 Done Done Sent false
82872024-02-04T14:03:59.729Z 14 Acked 1013 Write 8 Done Done Sent false
82882024-02-04T14:03:59.729Z 15 Acked 1014 Read 7 Done Done Sent false
82892024-02-04T14:03:59.729Z 16 Acked 1015 Read 1 Done Done Sent false
82902024-02-04T14:03:59.729Z 17 Acked 1016 Read 8 Done Done Sent false
82912024-02-04T14:03:59.729Z 18 Acked 1017 Read 5 Done Done Sent false
82922024-02-04T14:03:59.729Z 19 Acked 1018 Read 3 Done Done Sent false
82932024-02-04T14:03:59.729Z 20 Acked 1019 Write 8 Done Done Sent false
82942024-02-04T14:03:59.729Z 21 Acked 1020 Read 5 Done Done Sent false
82952024-02-04T14:03:59.729Z 22 Acked 1021 Read 8 Done Done Sent false
82962024-02-04T14:03:59.729Z 23 Acked 1022 Write 9 Done Done Sent false
82972024-02-04T14:03:59.729Z 24 Acked 1023 Write 5 Done Done Sent false
82982024-02-04T14:03:59.729Z 25 Acked 1024 Write 5 Done Done Sent false
82992024-02-04T14:03:59.729Z 26 Acked 1025 Read 5 Done Done Sent false
83002024-02-04T14:03:59.729Z 27 Acked 1026 Read 5 Done Done Sent false
83012024-02-04T14:03:59.729Z 28 Acked 1027 Write 1 Done Done Sent false
83022024-02-04T14:03:59.729Z 29 Acked 1028 Write 4 Done Done Sent false
83032024-02-04T14:03:59.729Z 30 Acked 1029 Write 5 Sent Sent Sent false
83042024-02-04T14:03:59.729Z STATES DS:0 DS:1 DS:2 TOTAL
83052024-02-04T14:03:59.729Z New 0 0 0 0
83062024-02-04T14:03:59.729Z Sent 1 1 26 28
83072024-02-04T14:03:59.730Z Done 29 29 4 62
83082024-02-04T14:03:59.730Z Skipped 0 0 0 0
83092024-02-04T14:03:59.730Z Error 0 0 0 0
83102024-02-04T14:03:59.730ZLast Flush: 1007 1007 0
83112024-02-04T14:03:59.730ZDownstairs last five completed:
83122024-02-04T14:03:59.730ZUpstairs last five completed: 30 29 28 27 26
83132024-02-04T14:03:59.730Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:03:58.761024664Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1433}
83142024-02-04T14:03:59.733ZWrote out file "/tmp/test_repair_verify.data"
83152024-02-04T14:03:59.736Z
83162024-02-04T14:03:59.738ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83172024-02-04T14:03:59.766ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83182024-02-04T14:03:59.766Z 0 000-019 70 70 70 110 110 110 F F F
83192024-02-04T14:03:59.766Z 1 020-039 74 74 74 117 117 117 F F F
83202024-02-04T14:03:59.766Z 2 040-059 75 75 75 120 120 120 F F F
83212024-02-04T14:03:59.766Z 3 060-079 73 73 73 116 116 116 F F F
83222024-02-04T14:03:59.766Z 4 080-099 68 68 68 107 107 107 F F F
83232024-02-04T14:03:59.766Z 5 100-119 74 74 74 117 117 117 F F F
83242024-02-04T14:03:59.766Z 6 120-139 64 64 64 101 101 101 F F F
83252024-02-04T14:03:59.769Z 7 140-159 76 76 68 121 121 107 F F F <---
83262024-02-04T14:03:59.770Z 8 160-179 76 76 73 121 121 116 F F F <---
83272024-02-04T14:03:59.770Z 9 180-199 70 70 70 110 110 110 F F F
83282024-02-04T14:03:59.770Z 10 200-219 70 70 70 109 109 109 T T F <---
83292024-02-04T14:03:59.770Z 11 220-239 74 74 74 117 117 117 F F F
83302024-02-04T14:03:59.770Z 12 240-259 74 74 74 119 119 119 F F F
83312024-02-04T14:03:59.770Z 13 260-279 73 73 73 116 116 116 F F F
83322024-02-04T14:03:59.770Z 14 280-299 70 70 70 110 110 110 F F F
83332024-02-04T14:03:59.770Z 15 300-319 74 74 74 119 119 119 F F F
83342024-02-04T14:03:59.770Z 16 320-339 64 64 64 101 101 101 F F F
83352024-02-04T14:03:59.770Z 17 340-359 76 76 74 121 121 117 F F T <---
83362024-02-04T14:03:59.770Z 18 360-379 74 74 74 117 117 117 T T F <---
83372024-02-04T14:03:59.770Z 19 380-399 72 72 72 112 112 112 T T F <---
83382024-02-04T14:03:59.770Z 20 400-419 75 75 75 120 120 120 T T F <---
83392024-02-04T14:03:59.770Z 21 420-439 74 74 74 118 118 118 T T F <---
83402024-02-04T14:03:59.770Z 22 440-459 73 73 73 116 116 116 T T F <---
83412024-02-04T14:03:59.770Z 23 460-479 66 66 66 103 103 103 F F F
83422024-02-04T14:03:59.770Z 24 480-499 72 72 72 115 115 115 T T F <---
83432024-02-04T14:03:59.770Z 25 500-519 76 76 74 121 121 117 T T F <---
83442024-02-04T14:03:59.770Z 26 520-539 72 72 72 113 113 113 T T F <---
83452024-02-04T14:03:59.770Z 27 540-559 74 74 74 118 118 118 T T F <---
83462024-02-04T14:03:59.770Z 28 560-579 74 74 74 117 117 117 T T F <---
83472024-02-04T14:03:59.770Z 29 580-599 70 70 70 109 109 109 F F F
83482024-02-04T14:03:59.770ZMax gen: 76, Max flush: 121
83492024-02-04T14:03:59.771ZError: Difference in extent metadata found!
83502024-02-04T14:03:59.773ZA Difference in extent metadata is expected here
83512024-02-04T14:03:59.774ZOn loop 38
83522024-02-04T14:03:59.774Z
83532024-02-04T14:03:59.774ZVerifying data now
83542024-02-04T14:04:04.984ZNo change after verify
83552024-02-04T14:04:04.984ZLoop: 38 Downstairs dump after verify (and repair):
83562024-02-04T14:04:05.017ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83572024-02-04T14:04:05.017Z 0 000-019 70 70 70 110 110 110 F F F
83582024-02-04T14:04:05.017Z 1 020-039 74 74 74 117 117 117 F F F
83592024-02-04T14:04:05.017Z 2 040-059 75 75 75 120 120 120 F F F
83602024-02-04T14:04:05.017Z 3 060-079 73 73 73 116 116 116 F F F
83612024-02-04T14:04:05.017Z 4 080-099 68 68 68 107 107 107 F F F
83622024-02-04T14:04:05.017Z 5 100-119 74 74 74 117 117 117 F F F
83632024-02-04T14:04:05.017Z 6 120-139 64 64 64 101 101 101 F F F
83642024-02-04T14:04:05.017Z 7 140-159 76 76 76 121 121 121 F F F
83652024-02-04T14:04:05.017Z 8 160-179 76 76 76 121 121 121 F F F
83662024-02-04T14:04:05.017Z 9 180-199 70 70 70 110 110 110 F F F
83672024-02-04T14:04:05.040Z 10 200-219 77 77 77 122 122 122 F F F
83682024-02-04T14:04:05.040Z 11 220-239 74 74 74 117 117 117 F F F
83692024-02-04T14:04:05.040Z 12 240-259 74 74 74 119 119 119 F F F
83702024-02-04T14:04:05.040Z 13 260-279 73 73 73 116 116 116 F F F
83712024-02-04T14:04:05.040Z 14 280-299 70 70 70 110 110 110 F F F
83722024-02-04T14:04:05.040Z 15 300-319 74 74 74 119 119 119 F F F
83732024-02-04T14:04:05.040Z 16 320-339 64 64 64 101 101 101 F F F
83742024-02-04T14:04:05.040Z 17 340-359 76 76 76 121 121 121 F F F
83752024-02-04T14:04:05.041Z 18 360-379 77 77 77 122 122 122 F F F
83762024-02-04T14:04:05.041Z 19 380-399 77 77 77 122 122 122 F F F
83772024-02-04T14:04:05.041Z 20 400-419 77 77 77 122 122 122 F F F
83782024-02-04T14:04:05.041Z 21 420-439 77 77 77 122 122 122 F F F
83792024-02-04T14:04:05.041Z 22 440-459 77 77 77 122 122 122 F F F
83802024-02-04T14:04:05.041Z 23 460-479 66 66 66 103 103 103 F F F
83812024-02-04T14:04:05.041Z 24 480-499 77 77 77 122 122 122 F F F
83822024-02-04T14:04:05.041Z 25 500-519 77 77 77 122 122 122 F F F
83832024-02-04T14:04:05.041Z 26 520-539 77 77 77 122 122 122 F F F
83842024-02-04T14:04:05.041Z 27 540-559 77 77 77 122 122 122 F F F
83852024-02-04T14:04:05.041Z 28 560-579 77 77 77 122 122 122 F F F
83862024-02-04T14:04:05.041Z 29 580-599 70 70 70 109 109 109 F F F
83872024-02-04T14:04:05.041ZMax gen: 77, Max flush: 122
83882024-02-04T14:04:05.041Z
83892024-02-04T14:04:05.041ZBegin loop 39
83902024-02-04T14:04:05.041ZDownstairs to restart: 0
83912024-02-04T14:04:05.047Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.073833384Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441}
83922024-02-04T14:04:05.047Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.074349803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441}
83932024-02-04T14:04:05.048Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.074425559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441}
83942024-02-04T14:04:05.048Z{"msg":"Crucible 716a047a-c5df-4779-a0d9-1df854278283 has session id: 6061310a-5073-4903-8b9f-243ff302f992","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.074469388Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
83952024-02-04T14:04:05.048Z{"msg"Crucible runtime is spawned
83962024-02-04T14:04:05.048Z:"Upstairs opts: Upstairs UUID: 716a047a-c5df-4779-a0d9-1df854278283, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.074609516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
83972024-02-04T14:04:05.048Z{"msg":"Crucible stats registered with UUID: 716a047a-c5df-4779-a0d9-1df854278283","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.074673526Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
83982024-02-04T14:04:05.048Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.074699034Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441}
83992024-02-04T14:04:05.048Z{"msg":"716a047a-c5df-4779-a0d9-1df854278283 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.07478983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84002024-02-04T14:04:05.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.074908749Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84012024-02-04T14:04:05.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075046136Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84022024-02-04T14:04:05.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075278976Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84032024-02-04T14:04:05.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075512014Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84042024-02-04T14:04:05.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075695572Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84052024-02-04T14:04:05.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075770885Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84062024-02-04T14:04:05.049Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075800853Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"io task","client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84072024-02-04T14:04:05.049Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075840485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"io task","client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84082024-02-04T14:04:05.049Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075912787Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"io task","client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84092024-02-04T14:04:05.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.075985472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"io task","client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84102024-02-04T14:04:05.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.076056262Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"io task","client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84112024-02-04T14:04:05.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.076134746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"io task","client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84122024-02-04T14:04:05.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.076713198Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84132024-02-04T14:04:05.049Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.076791667Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84142024-02-04T14:04:05.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.076834746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84152024-02-04T14:04:05.049Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.076914753Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84162024-02-04T14:04:05.049Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.07765462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84172024-02-04T14:04:05.050Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.077798061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84182024-02-04T14:04:05.050Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.077970792Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84192024-02-04T14:04:05.050Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.078025376Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84202024-02-04T14:04:05.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.078565249Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84212024-02-04T14:04:05.050Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.078733121Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84222024-02-04T14:04:05.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.078813308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84232024-02-04T14:04:05.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.078840161Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84242024-02-04T14:04:05.050Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.078910242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84252024-02-04T14:04:05.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.078939068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84262024-02-04T14:04:05.050Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.079252922Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84272024-02-04T14:04:05.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080020068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84282024-02-04T14:04:05.055Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080098557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84292024-02-04T14:04:05.055Z{"msg":"[0]R flush_numbers[0..12]: [110, 117, 120, 116, 107, 117, 101, 121, 121, 110, 122, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.08013838Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84302024-02-04T14:04:05.055Z{"msg":"[0]R generation[0..12]: [70, 74, 75, 73, 68, 74, 64, 76, 76, 70, 77, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080196197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84312024-02-04T14:04:05.055Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080229653Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84322024-02-04T14:04:05.055Z{"msg":"[1]R flush_numbers[0..12]: [110, 117, 120, 116, 107, 117, 101, 121, 121, 110, 122, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080287796Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84332024-02-04T14:04:05.056Z{"msg":"[1]R generation[0..12]: [70, 74, 75, 73, 68, 74, 64, 76, 76, 70, 77, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080318679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84342024-02-04T14:04:05.056Z{"msg":"Wait for a query_work_queue command to finish before sending IO
84352024-02-04T14:04:05.056Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080379038Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84362024-02-04T14:04:05.056Z{"msg":"[2]R flush_numbers[0..12]: [110, 117, 120, 116, 107, 117, 101, 121, 121, 110, 122, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080443179Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84372024-02-04T14:04:05.056Z{"msg":"[2]R generation[0..12]: [70, 74, 75, 73, 68, 74, 64, 76, 76, 70, 77, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080475495Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84382024-02-04T14:04:05.056Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080531492Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84392024-02-04T14:04:05.056Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.08055689Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84402024-02-04T14:04:05.056Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080613952Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84412024-02-04T14:04:05.056Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080639913Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84422024-02-04T14:04:05.056Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080696172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84432024-02-04T14:04:05.056Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080760519Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84442024-02-04T14:04:05.056Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080784457Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84452024-02-04T14:04:05.056Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080856494Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84462024-02-04T14:04:05.057Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080920816Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84472024-02-04T14:04:05.057Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.080961397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84482024-02-04T14:04:05.057Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.08102673Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84492024-02-04T14:04:05.057Z{"msg":"716a047a-c5df-4779-a0d9-1df854278283 is now active with session: 6061310a-5073-4903-8b9f-243ff302f992","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.081068339Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84502024-02-04T14:04:05.057Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.08115792Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441,"session_id":"6061310a-5073-4903-8b9f-243ff302f992"}
84512024-02-04T14:04:05.057Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:04.081198577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441}
84522024-02-04T14:04:05.057ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84532024-02-04T14:04:05.057ZLoading write count information from file "/tmp/test_repair_verify.data"
84542024-02-04T14:04:05.057ZRun Repair workload
84552024-02-04T14:04:05.057Z01/30 Write block 131 len 8192 data: 3 3
84562024-02-04T14:04:05.057Z02/30 Write block 132 len 4096 data: 4
84572024-02-04T14:04:05.057Z03/30 Write block 204 len 32768 data: 12 9 8 7 5 3 3 5
84582024-02-04T14:04:05.062Z04/30 Write block 125 len 32768 data: 7 8 7 6 4 4 4 5
84592024-02-04T14:04:05.062Z05/30 Flush
84602024-02-04T14:04:05.062Z06/30 Write block 295 len 16384 data: 5 4 4 5
84612024-02-04T14:04:05.062Z07/30 Write block 73 len 4096 data: 7
84622024-02-04T14:04:05.065Z08/30 Write block 565 len 20480 data: 8 8 6 4 5
84632024-02-04T14:04:05.065Z09/30 Write block 197 len 4096 data: 6
84642024-02-04T14:04:05.065Z10/30 Read block 150 len 40960
84652024-02-04T14:04:05.069Z11/30 Read block 229 len 4096
84662024-02-04T14:04:05.069Z12/30 Read block 492 len 24576
84672024-02-04T14:04:05.069Z13/30 Read block 336 len 40960
84682024-02-04T14:04:05.072Z14/30 Read block 381 len 4096
84692024-02-04T14:04:05.072Z15/30 Write block 160 len 16384 data: 5 6 6 6
84702024-02-04T14:04:05.072Z16/30 Write block 579 len 8192 data: 11 13
84712024-02-04T14:04:05.075Z17/30 Read block 309 len 4096
84722024-02-04T14:04:05.076Z18/30 Write block 338 len 32768 data: 5 5 5 6 5 6 6 6
84732024-02-04T14:04:05.076Z19/30 Read block 67 len 40960
84742024-02-04T14:04:05.080Z20/30 Flush
84752024-02-04T14:04:05.084Z21/30 Write block 451 len 40960 data: 7 8 7 9 8 9 9 8 8 6
84762024-02-04T14:04:05.084Z22/30 Flush
84772024-02-04T14:04:05.092Z23/30 Flush
84782024-02-04T14:04:05.092Z24/30 Write block 483 len 12288 data: 3 4 6
84792024-02-04T14:04:05.092Z25/30 Read block 6 len 4096
84802024-02-04T14:04:05.092Z26/30 Write block 395 len 24576 data: 9 9 11 9 9 10
84812024-02-04T14:04:05.096Z27/30 Write block 442 len 36864 data: 5 6 7 7 7 7 6 6 7
84822024-02-04T14:04:05.096Z28/30 Read block 230 len 32768
84832024-02-04T14:04:05.096Z29/30 Read block 594 len 4096
84842024-02-04T14:04:05.096Z30/30 Read block 351 len 24576
84852024-02-04T14:04:05.096Z----------------------------------------------------------------
84862024-02-04T14:04:05.096Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84872024-02-04T14:04:05.096ZStates: Active Active Active
84882024-02-04T14:04:05.096ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84892024-02-04T14:04:05.096Z 1 Acked 1000 Write 2 Done Done Done false
84902024-02-04T14:04:05.096Z 2 Acked 1001 Write 1 Sent Done Done false
84912024-02-04T14:04:05.096Z 3 Acked 1002 Write 8 Sent Done Done false
84922024-02-04T14:04:05.096Z 4 Acked 1003 Write 8 Sent Done Done false
84932024-02-04T14:04:05.097Z 5 Acked 1004 Flush 0 Sent Done Done false
84942024-02-04T14:04:05.097Z 6 Acked 1005 Write 4 Sent Done Done false
84952024-02-04T14:04:05.097Z 7 Acked 1006 Write 1 Sent Done Done false
84962024-02-04T14:04:05.097Z 8 Acked 1007 Write 5 Sent Done Done false
84972024-02-04T14:04:05.097Z 9 Acked 1008 Write 1 Sent Done Done false
84982024-02-04T14:04:05.097Z 10 Acked 1009 Read 10 Sent Done Done false
84992024-02-04T14:04:05.097Z 11 Acked 1010 Read 1 Sent Done Done false
85002024-02-04T14:04:05.097Z 12 Acked 1011 Read 6 Sent Done Done false
85012024-02-04T14:04:05.097Z 13 Acked 1012 Read 10 Sent Done Done false
85022024-02-04T14:04:05.097Z 14 Acked 1013 Read 1 Sent Done Done false
85032024-02-04T14:04:05.097Z 15 Acked 1014 Write 4 Sent Done Done false
85042024-02-04T14:04:05.097Z 16 Acked 1015 Write 2 Sent Done Done false
85052024-02-04T14:04:05.097Z 17 Acked 1016 Read 1 Sent Done Done false
85062024-02-04T14:04:05.097Z 18 Acked 1017 Write 8 Sent Done Done false
85072024-02-04T14:04:05.098Z 19 Acked 1018 Read 10 Sent Done Done false
85082024-02-04T14:04:05.098Z 20 Acked 1019 Flush 0 Sent Done Done false
85092024-02-04T14:04:05.098Z 21 Acked 1020 Write 10 Sent Done Done false
85102024-02-04T14:04:05.103Z 22 Acked 1021 Flush 0 Sent Done Done false
85112024-02-04T14:04:05.103Z 23 Acked 1022 Flush 0 Sent Done Done false
85122024-02-04T14:04:05.103Z 24 Acked 1023 Write 3 Sent Done Done false
85132024-02-04T14:04:05.104Z 25 Acked 1024 Read 1 Sent Done Done false
85142024-02-04T14:04:05.104Z 26 Acked 1025 Write 6 Sent Done Done false
85152024-02-04T14:04:05.104Z 27 Acked 1026 Write 9 Sent Done Done false
85162024-02-04T14:04:05.104Z 28 Acked 1027 Read 8 Sent Done Done false
85172024-02-04T14:04:05.104Z 29 Acked 1028 Read 1 Sent Done Done false
85182024-02-04T14:04:05.104Z 30 Acked 1029 Read 6 Sent Done Sent false
85192024-02-04T14:04:05.104Z STATES DS:0 DS:1 DS:2 TOTAL
85202024-02-04T14:04:05.104Z New 0 0 0 0
85212024-02-04T14:04:05.104Z Sent 29 0 1 30
85222024-02-04T14:04:05.104Z Done 1 30 29 60
85232024-02-04T14:04:05.104Z Skipped 0 0 0 0
85242024-02-04T14:04:05.104Z Error 0 0 0 0
85252024-02-04T14:04:05.105ZLast Flush: 0 1022 1022
85262024-02-04T14:04:05.105ZDownstairs last five completed:
85272024-02-04T14:04:05.105ZUpstairs last five completed: 30 29 28 27 26
85282024-02-04T14:04:05.105Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:04.129340986Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1441}
85292024-02-04T14:04:05.105ZWrote out file "/tmp/test_repair_verify.data"
85302024-02-04T14:04:05.108Z
85312024-02-04T14:04:05.111ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85322024-02-04T14:04:05.135ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85332024-02-04T14:04:05.135Z 0 000-019 70 70 70 110 110 110 F F F
85342024-02-04T14:04:05.135Z 1 020-039 74 74 74 117 117 117 F F F
85352024-02-04T14:04:05.135Z 2 040-059 75 75 75 120 120 120 F F F
85362024-02-04T14:04:05.135Z 3 060-079 73 78 78 116 124 124 F F F <---
85372024-02-04T14:04:05.136Z 4 080-099 68 68 68 107 107 107 F F F
85382024-02-04T14:04:05.136Z 5 100-119 74 74 74 117 117 117 F F F
85392024-02-04T14:04:05.136Z 6 120-139 64 78 78 101 123 123 T F F <---
85402024-02-04T14:04:05.136Z 7 140-159 76 76 76 121 121 121 F F F
85412024-02-04T14:04:05.136Z 8 160-179 76 78 78 121 124 124 F F F <---
85422024-02-04T14:04:05.136Z 9 180-199 70 78 78 110 124 124 F F F <---
85432024-02-04T14:04:05.136Z 10 200-219 77 78 78 122 123 123 F F F <---
85442024-02-04T14:04:05.136Z 11 220-239 74 74 74 117 117 117 F F F
85452024-02-04T14:04:05.142Z 12 240-259 74 74 74 119 119 119 F F F
85462024-02-04T14:04:05.142Z 13 260-279 73 73 73 116 116 116 F F F
85472024-02-04T14:04:05.142Z 14 280-299 70 78 78 110 124 124 F F F <---
85482024-02-04T14:04:05.142Z 15 300-319 74 74 74 119 119 119 F F F
85492024-02-04T14:04:05.142Z 16 320-339 64 78 78 101 124 124 F F F <---
85502024-02-04T14:04:05.142Z 17 340-359 76 78 78 121 124 124 F F F <---
85512024-02-04T14:04:05.142Z 18 360-379 77 77 77 122 122 122 F F F
85522024-02-04T14:04:05.142Z 19 380-399 77 77 77 122 122 122 F T T <---
85532024-02-04T14:04:05.143Z 20 400-419 77 77 77 122 122 122 F T T <---
85542024-02-04T14:04:05.143Z 21 420-439 77 77 77 122 122 122 F F F
85552024-02-04T14:04:05.143Z 22 440-459 77 78 78 122 125 125 F T T <---
85562024-02-04T14:04:05.143Z 23 460-479 66 78 78 103 125 125 F F F <---
85572024-02-04T14:04:05.143Z 24 480-499 77 77 77 122 122 122 F T T <---
85582024-02-04T14:04:05.143Z 25 500-519 77 77 77 122 122 122 F F F
85592024-02-04T14:04:05.143Z 26 520-539 77 77 77 122 122 122 F F F
85602024-02-04T14:04:05.143Z 27 540-559 77 77 77 122 122 122 F F F
85612024-02-04T14:04:05.143Z 28 560-579 77 78 78 122 124 124 F F F <---
85622024-02-04T14:04:05.143Z 29 580-599 70 78 78 109 124 124 F F F <---
85632024-02-04T14:04:05.143ZMax gen: 78, Max flush: 125
85642024-02-04T14:04:05.143ZError: Difference in extent metadata found!
85652024-02-04T14:04:05.144ZA Difference in extent metadata is expected here
85662024-02-04T14:04:05.144ZOn loop 39
85672024-02-04T14:04:05.144Z
85682024-02-04T14:04:05.147ZVerifying data now
85692024-02-04T14:04:10.413ZNo change after verify
85702024-02-04T14:04:10.413ZLoop: 39 Downstairs dump after verify (and repair):
85712024-02-04T14:04:10.436ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85722024-02-04T14:04:10.436Z 0 000-019 70 70 70 110 110 110 F F F
85732024-02-04T14:04:10.436Z 1 020-039 74 74 74 117 117 117 F F F
85742024-02-04T14:04:10.436Z 2 040-059 75 75 75 120 120 120 F F F
85752024-02-04T14:04:10.436Z 3 060-079 78 78 78 124 124 124 F F F
85762024-02-04T14:04:10.436Z 4 080-099 68 68 68 107 107 107 F F F
85772024-02-04T14:04:10.436Z 5 100-119 74 74 74 117 117 117 F F F
85782024-02-04T14:04:10.440Z 6 120-139 78 78 78 123 123 123 F F F
85792024-02-04T14:04:10.440Z 7 140-159 76 76 76 121 121 121 F F F
85802024-02-04T14:04:10.440Z 8 160-179 78 78 78 124 124 124 F F F
85812024-02-04T14:04:10.440Z 9 180-199 78 78 78 124 124 124 F F F
85822024-02-04T14:04:10.440Z 10 200-219 78 78 78 123 123 123 F F F
85832024-02-04T14:04:10.440Z 11 220-239 74 74 74 117 117 117 F F F
85842024-02-04T14:04:10.441Z 12 240-259 74 74 74 119 119 119 F F F
85852024-02-04T14:04:10.441Z 13 260-279 73 73 73 116 116 116 F F F
85862024-02-04T14:04:10.441Z 14 280-299 78 78 78 124 124 124 F F F
85872024-02-04T14:04:10.441Z 15 300-319 74 74 74 119 119 119 F F F
85882024-02-04T14:04:10.441Z 16 320-339 78 78 78 124 124 124 F F F
85892024-02-04T14:04:10.441Z 17 340-359 78 78 78 124 124 124 F F F
85902024-02-04T14:04:10.441Z 18 360-379 77 77 77 122 122 122 F F F
85912024-02-04T14:04:10.441Z 19 380-399 79 79 79 126 126 126 F F F
85922024-02-04T14:04:10.441Z 20 400-419 79 79 79 126 126 126 F F F
85932024-02-04T14:04:10.441Z 21 420-439 77 77 77 122 122 122 F F F
85942024-02-04T14:04:10.441Z 22 440-459 79 79 79 126 126 126 F F F
85952024-02-04T14:04:10.441Z 23 460-479 78 78 78 125 125 125 F F F
85962024-02-04T14:04:10.441Z 24 480-499 79 79 79 126 126 126 F F F
85972024-02-04T14:04:10.441Z 25 500-519 77 77 77 122 122 122 F F F
85982024-02-04T14:04:10.441Z 26 520-539 77 77 77 122 122 122 F F F
85992024-02-04T14:04:10.441Z 27 540-559 77 77 77 122 122 122 F F F
86002024-02-04T14:04:10.441Z 28 560-579 78 78 78 124 124 124 F F F
86012024-02-04T14:04:10.441Z 29 580-599 78 78 78 124 124 124 F F F
86022024-02-04T14:04:10.441ZMax gen: 79, Max flush: 126
86032024-02-04T14:04:10.441Z
86042024-02-04T14:04:10.441ZBegin loop 40
86052024-02-04T14:04:10.441ZDownstairs to restart: 2
86062024-02-04T14:04:10.470Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.503025144Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449}
86072024-02-04T14:04:10.470Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.503506496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449}
86082024-02-04T14:04:10.476Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.503584183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449}
86092024-02-04T14:04:10.476Z{"msg":"Crucible e0f4e5e3-b1b5-42f9-bf59-dca3b5c38279 has session id: 01e407f5-1834-4b17-ac74-46b193513e06","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.503652429Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86102024-02-04T14:04:10.476Z{"msg":"Upstairs opts: Upstairs UUID: e0f4e5e3-b1b5-42f9-bf59-dca3b5c38279, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
86112024-02-04T14:04:10.476Z,"time":"2024-02-04T14:04:09.503702357Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86122024-02-04T14:04:10.476Z{"msg":"Crucible stats registered with UUID: e0f4e5e3-b1b5-42f9-bf59-dca3b5c38279","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.503808827Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86132024-02-04T14:04:10.476Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.503907744Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449}
86142024-02-04T14:04:10.476Z{"msg":"e0f4e5e3-b1b5-42f9-bf59-dca3b5c38279 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.503973925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86152024-02-04T14:04:10.476Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504014604Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86162024-02-04T14:04:10.476Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.50405866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86172024-02-04T14:04:10.476Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504107059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86182024-02-04T14:04:10.476Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504150406Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86192024-02-04T14:04:10.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504189467Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86202024-02-04T14:04:10.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504230938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86212024-02-04T14:04:10.477Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504297339Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"io task","client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86222024-02-04T14:04:10.477Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504341784Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"io task","client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86232024-02-04T14:04:10.477Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504551229Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"io task","client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86242024-02-04T14:04:10.477Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504748441Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"io task","client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86252024-02-04T14:04:10.477Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504799041Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"io task","client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86262024-02-04T14:04:10.477Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.504878048Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"io task","client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86272024-02-04T14:04:10.477Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.505949869Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86282024-02-04T14:04:10.477Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.50611472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86292024-02-04T14:04:10.477Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.506195786Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86302024-02-04T14:04:10.477Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.506321382Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86312024-02-04T14:04:10.477Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.506981011Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86322024-02-04T14:04:10.482Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.507234169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86332024-02-04T14:04:10.482Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.507320011Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86342024-02-04T14:04:10.482Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.507350713Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86352024-02-04T14:04:10.482Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.507748951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86362024-02-04T14:04:10.482Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.507816091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86372024-02-04T14:04:10.482Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.507843231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86382024-02-04T14:04:10.482Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.50814369Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86392024-02-04T14:04:10.483Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.50820265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86402024-02-04T14:04:10.483Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.508227542Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86412024-02-04T14:04:10.483Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.508268774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86422024-02-04T14:04:10.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.508942115Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86432024-02-04T14:04:10.483Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509049094Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86442024-02-04T14:04:10.483Z{"msg":"[0]R flush_numbers[0..12]: [110, 117, 120, 124, 107, 117, 123, 121, 124, 124, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509123805Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86452024-02-04T14:04:10.483Z{"msg":"[0]R generation[0..12]: [70, 74, 75, 78, 68, 74, 78, 76, 78, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509167475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86462024-02-04T14:04:10.483Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
86472024-02-04T14:04:10.483Z,"time":"2024-02-04T14:04:09.509249996Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86482024-02-04T14:04:10.483Z{"msg":"[1]R flush_numbers[0..12]: [110, 117, 120, 124, 107, 117, 123, 121, 124, 124, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509351683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86492024-02-04T14:04:10.483Z{"msg":"[1]R generation[0..12]: [70, 74, 75, 78, 68, 74, 78, 76, 78, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509421795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86502024-02-04T14:04:10.483Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509467839Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86512024-02-04T14:04:10.483Z{"msg":"[2]R flush_numbers[0..12]: [110, 117, 120, 124, 107, 117, 123, 121, 124, 124, 123, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509525967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86522024-02-04T14:04:10.483Z{"msg":"[2]R generation[0..12]: [70, 74, 75, 78, 68, 74, 78, 76, 78, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509600981Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86532024-02-04T14:04:10.483Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509637042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86542024-02-04T14:04:10.483Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509660196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86552024-02-04T14:04:10.483Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509702142Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86562024-02-04T14:04:10.483Z{"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509725557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86572024-02-04T14:04:10.483Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509747912Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86582024-02-04T14:04:10.483Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509792606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86592024-02-04T14:04:10.483Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.50981423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86602024-02-04T14:04:10.483Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.50985476Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86612024-02-04T14:04:10.484Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509878119Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86622024-02-04T14:04:10.484Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509925509Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86632024-02-04T14:04:10.484Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509947636Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86642024-02-04T14:04:10.484Z{"msg":"e0f4e5e3-b1b5-42f9-bf59-dca3b5c38279 is now active with session: 01e407f5-1834-4b17-ac74-46b193513e06","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.509991122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86652024-02-04T14:04:10.484Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.510025618Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449,"session_id":"01e407f5-1834-4b17-ac74-46b193513e06"}
86662024-02-04T14:04:10.484Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:09.510086518Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449}
86672024-02-04T14:04:10.484ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86682024-02-04T14:04:10.484ZLoading write count information from file "/tmp/test_repair_verify.data"
86692024-02-04T14:04:10.484ZRun Repair workload
86702024-02-04T14:04:10.484Z01/30 Write block 411 len 32768 data: 8 7 9 9 9 7 8 7
86712024-02-04T14:04:10.484Z02/30 Write block 283 len 32768 data: 8 13 11 9 10 10 9 10
86722024-02-04T14:04:10.487Z03/30 Read block 135 len 32768
86732024-02-04T14:04:10.487Z04/30 Write block 370 len 8192 data: 8 7
86742024-02-04T14:04:10.487Z05/30 Write block 589 len 32768 data: 5 6 5 6 6 6 6 3
86752024-02-04T14:04:10.487Z06/30 Write block 242 len 32768 data: 4 4 4 4 4 6 8 11
86762024-02-04T14:04:10.491Z07/30 Read block 542 len 28672
86772024-02-04T14:04:10.491Z08/30 Write block 1 len 40960 data: 3 4 4 6 6 6 6 6 7 4
86782024-02-04T14:04:10.508Z09/30 Read block 13 len 8192
86792024-02-04T14:04:10.508Z10/30 Write block 452 len 32768 data: 9 8 10 9 10 10 9 9
86802024-02-04T14:04:10.508Z11/30 Read block 98 len 12288
86812024-02-04T14:04:10.512Z12/30 Write block 217 len 4096 data: 9
86822024-02-04T14:04:10.512Z13/30 Flush
86832024-02-04T14:04:10.512Z14/30 Write block 414 len 4096 data: 10
86842024-02-04T14:04:10.512Z15/30 Write block 576 len 12288 data: 10 10 9
86852024-02-04T14:04:10.512Z16/30 Read block 579 len 28672
86862024-02-04T14:04:10.512Z17/30 Write block 459 len 8192 data: 10 7
86872024-02-04T14:04:10.512Z18/30 Read block 220 len 40960
86882024-02-04T14:04:10.512Z19/30 Write block 405 len 24576 data: 8 7 7 6 6 6
86892024-02-04T14:04:10.512Z20/30 Read block 81 len 32768
86902024-02-04T14:04:10.512Z21/30 Read block 287 len 40960
86912024-02-04T14:04:10.512Z22/30 Read block 240 len 4096
86922024-02-04T14:04:10.516Z23/30 Read block 14 len 24576
86932024-02-04T14:04:10.516Z24/30 Read block 35 len 32768
86942024-02-04T14:04:10.516Z25/30 Write block 254 len 12288 data: 9 9 8
86952024-02-04T14:04:10.516Z26/30 Read block 535 len 12288
86962024-02-04T14:04:10.520Z27/30 Write block 594 len 12288 data: 7 7 4
86972024-02-04T14:04:10.520Z28/30 Read block 549 len 36864
86982024-02-04T14:04:10.523Z29/30 Write block 78 len 36864 data: 7 7 6 7 7 5 6 7 6
86992024-02-04T14:04:10.523Z30/30 Read block 510 len 16384
87002024-02-04T14:04:10.527Z----------------------------------------------------------------
87012024-02-04T14:04:10.527Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
87022024-02-04T14:04:10.527ZStates: Active Active Active
87032024-02-04T14:04:10.527ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87042024-02-04T14:04:10.527Z 1 Acked 1000 Write 8 Done Done Sent false
87052024-02-04T14:04:10.527Z 2 Acked 1001 Write 8 Done Done Sent false
87062024-02-04T14:04:10.527Z 3 Acked 1002 Read 8 Done Done Sent false
87072024-02-04T14:04:10.527Z 4 Acked 1003 Write 2 Done Done Sent false
87082024-02-04T14:04:10.527Z 5 Acked 1004 Write 8 Done Done Sent false
87092024-02-04T14:04:10.527Z 6 Acked 1005 Write 8 Done Done Sent false
87102024-02-04T14:04:10.527Z 7 Acked 1006 Read 7 Done Done Sent false
87112024-02-04T14:04:10.527Z 8 Acked 1007 Write 10 Done Done Sent false
87122024-02-04T14:04:10.527Z 9 Acked 1008 Read 2 Done Done Sent false
87132024-02-04T14:04:10.527Z 10 Acked 1009 Write 8 Done Done Sent false
87142024-02-04T14:04:10.528Z 11 Acked 1010 Read 3 Done Done Sent false
87152024-02-04T14:04:10.528Z 12 Acked 1011 Write 1 Done Done Sent false
87162024-02-04T14:04:10.528Z 13 Acked 1012 Flush 0 Done Done Sent false
87172024-02-04T14:04:10.528Z 14 Acked 1013 Write 1 Done Done Sent false
87182024-02-04T14:04:10.528Z 15 Acked 1014 Write 3 Done Done Sent false
87192024-02-04T14:04:10.528Z 16 Acked 1015 Read 7 Done Done Sent false
87202024-02-04T14:04:10.528Z 17 Acked 1016 Write 2 Done Done Sent false
87212024-02-04T14:04:10.528Z 18 Acked 1017 Read 10 Done Done Sent false
87222024-02-04T14:04:10.528Z 19 Acked 1018 Write 6 Done Done Sent false
87232024-02-04T14:04:10.528Z 20 Acked 1019 Read 8 Done Done Sent false
87242024-02-04T14:04:10.528Z 21 Acked 1020 Read 10 Done Done Sent false
87252024-02-04T14:04:10.528Z 22 Acked 1021 Read 1 Done Done Sent false
87262024-02-04T14:04:10.528Z 23 Acked 1022 Read 6 Done Done Sent false
87272024-02-04T14:04:10.528Z 24 Acked 1023 Read 8 Done Done Sent false
87282024-02-04T14:04:10.528Z 25 Acked 1024 Write 3 Done Done Sent false
87292024-02-04T14:04:10.528Z 26 Acked 1025 Read 3 Done Done Sent false
87302024-02-04T14:04:10.528Z 27 Acked 1026 Write 3 Done Done Sent false
87312024-02-04T14:04:10.528Z 28 Acked 1027 Read 9 Done Done Sent false
87322024-02-04T14:04:10.528Z 29 Acked 1028 Write 9 Done Done Sent false
87332024-02-04T14:04:10.528Z 30 Acked 1029 Read 4 Sent Done Sent false
87342024-02-04T14:04:10.528Z STATES DS:0 DS:1 DS:2 TOTAL
87352024-02-04T14:04:10.528Z New 0 0 0 0
87362024-02-04T14:04:10.528Z Sent 1 0 30 31
87372024-02-04T14:04:10.528Z Done 29 30 0 59
87382024-02-04T14:04:10.528Z Skipped 0 0 0 0
87392024-02-04T14:04:10.529Z Error 0 0 0 0
87402024-02-04T14:04:10.529ZLast Flush: 1012 1012 0
87412024-02-04T14:04:10.529ZDownstairs last five completed:
87422024-02-04T14:04:10.529ZUpstairs last five completed: 30 29 28 27 26
87432024-02-04T14:04:10.529Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:09.556947959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1449}
87442024-02-04T14:04:10.529ZWrote out file "/tmp/test_repair_verify.data"
87452024-02-04T14:04:10.533Z
87462024-02-04T14:04:10.537ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87472024-02-04T14:04:10.564ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87482024-02-04T14:04:10.564Z 0 000-019 80 80 70 127 127 110 F F F <---
87492024-02-04T14:04:10.564Z 1 020-039 74 74 74 117 117 117 F F F
87502024-02-04T14:04:10.564Z 2 040-059 75 75 75 120 120 120 F F F
87512024-02-04T14:04:10.564Z 3 060-079 78 78 78 124 124 124 T T F <---
87522024-02-04T14:04:10.569Z 4 080-099 68 68 68 107 107 107 T T F <---
87532024-02-04T14:04:10.569Z 5 100-119 74 74 74 117 117 117 F F F
87542024-02-04T14:04:10.569Z 6 120-139 78 78 78 123 123 123 F F F
87552024-02-04T14:04:10.569Z 7 140-159 76 76 76 121 121 121 F F F
87562024-02-04T14:04:10.570Z 8 160-179 78 78 78 124 124 124 F F F
87572024-02-04T14:04:10.570Z 9 180-199 78 78 78 124 124 124 F F F
87582024-02-04T14:04:10.570Z 10 200-219 80 80 78 127 127 123 F F F <---
87592024-02-04T14:04:10.570Z 11 220-239 74 74 74 117 117 117 F F F
87602024-02-04T14:04:10.570Z 12 240-259 80 80 74 127 127 119 T T F <---
87612024-02-04T14:04:10.570Z 13 260-279 73 73 73 116 116 116 F F F
87622024-02-04T14:04:10.570Z 14 280-299 80 80 78 127 127 124 F F F <---
87632024-02-04T14:04:10.570Z 15 300-319 74 74 74 119 119 119 F F F
87642024-02-04T14:04:10.570Z 16 320-339 78 78 78 124 124 124 F F F
87652024-02-04T14:04:10.570Z 17 340-359 78 78 78 124 124 124 F F F
87662024-02-04T14:04:10.570Z 18 360-379 80 80 77 127 127 122 F F F <---
87672024-02-04T14:04:10.570Z 19 380-399 79 79 79 126 126 126 F F F
87682024-02-04T14:04:10.570Z 20 400-419 80 80 79 127 127 126 T T F <---
87692024-02-04T14:04:10.570Z 21 420-439 77 77 77 122 122 122 F F F
87702024-02-04T14:04:10.571Z 22 440-459 80 80 79 127 127 126 T T F <---
87712024-02-04T14:04:10.571Z 23 460-479 78 78 78 125 125 125 T T F <---
87722024-02-04T14:04:10.571Z 24 480-499 79 79 79 126 126 126 F F F
87732024-02-04T14:04:10.571Z 25 500-519 77 77 77 122 122 122 F F F
87742024-02-04T14:04:10.571Z 26 520-539 77 77 77 122 122 122 F F F
87752024-02-04T14:04:10.571Z 27 540-559 77 77 77 122 122 122 F F F
87762024-02-04T14:04:10.571Z 28 560-579 78 78 78 124 124 124 T T F <---
87772024-02-04T14:04:10.571Z 29 580-599 80 80 78 127 127 124 T T F <---
87782024-02-04T14:04:10.571ZMax gen: 80, Max flush: 127
87792024-02-04T14:04:10.571ZError: Difference in extent metadata found!
87802024-02-04T14:04:10.575ZA Difference in extent metadata is expected here
87812024-02-04T14:04:10.575ZOn loop 40
87822024-02-04T14:04:10.575Z
87832024-02-04T14:04:10.575ZVerifying data now
87842024-02-04T14:04:15.627ZNo change after verify
87852024-02-04T14:04:15.627ZLoop: 40 Downstairs dump after verify (and repair):
87862024-02-04T14:04:15.653ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87872024-02-04T14:04:15.653Z 0 000-019 80 80 80 127 127 127 F F F
87882024-02-04T14:04:15.653Z 1 020-039 74 74 74 117 117 117 F F F
87892024-02-04T14:04:15.653Z 2 040-059 75 75 75 120 120 120 F F F
87902024-02-04T14:04:15.653Z 3 060-079 81 81 81 128 128 128 F F F
87912024-02-04T14:04:15.653Z 4 080-099 81 81 81 128 128 128 F F F
87922024-02-04T14:04:15.653Z 5 100-119 74 74 74 117 117 117 F F F
87932024-02-04T14:04:15.653Z 6 120-139 78 78 78 123 123 123 F F F
87942024-02-04T14:04:15.653Z 7 140-159 76 76 76 121 121 121 F F F
87952024-02-04T14:04:15.654Z 8 160-179 78 78 78 124 124 124 F F F
87962024-02-04T14:04:15.654Z 9 180-199 78 78 78 124 124 124 F F F
87972024-02-04T14:04:15.654Z 10 200-219 80 80 80 127 127 127 F F F
87982024-02-04T14:04:15.654Z 11 220-239 74 74 74 117 117 117 F F F
87992024-02-04T14:04:15.654Z 12 240-259 81 81 81 128 128 128 F F F
88002024-02-04T14:04:15.654Z 13 260-279 73 73 73 116 116 116 F F F
88012024-02-04T14:04:15.654Z 14 280-299 80 80 80 127 127 127 F F F
88022024-02-04T14:04:15.657Z 15 300-319 74 74 74 119 119 119 F F F
88032024-02-04T14:04:15.657Z 16 320-339 78 78 78 124 124 124 F F F
88042024-02-04T14:04:15.657Z 17 340-359 78 78 78 124 124 124 F F F
88052024-02-04T14:04:15.657Z 18 360-379 80 80 80 127 127 127 F F F
88062024-02-04T14:04:15.657Z 19 380-399 79 79 79 126 126 126 F F F
88072024-02-04T14:04:15.657Z 20 400-419 81 81 81 128 128 128 F F F
88082024-02-04T14:04:15.657Z 21 420-439 77 77 77 122 122 122 F F F
88092024-02-04T14:04:15.657Z 22 440-459 81 81 81 128 128 128 F F F
88102024-02-04T14:04:15.657Z 23 460-479 81 81 81 128 128 128 F F F
88112024-02-04T14:04:15.657Z 24 480-499 79 79 79 126 126 126 F F F
88122024-02-04T14:04:15.657Z 25 500-519 77 77 77 122 122 122 F F F
88132024-02-04T14:04:15.657Z 26 520-539 77 77 77 122 122 122 F F F
88142024-02-04T14:04:15.657Z 27 540-559 77 77 77 122 122 122 F F F
88152024-02-04T14:04:15.658Z 28 560-579 81 81 81 128 128 128 F F F
88162024-02-04T14:04:15.658Z 29 580-599 81 81 81 128 128 128 F F F
88172024-02-04T14:04:15.658ZMax gen: 81, Max flush: 128
88182024-02-04T14:04:15.658Z
88192024-02-04T14:04:15.658ZBegin loop 41
88202024-02-04T14:04:15.658ZDownstairs to restart: 2
88212024-02-04T14:04:15.684Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.716684234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457}
88222024-02-04T14:04:15.684Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717169998Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457}
88232024-02-04T14:04:15.687Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717226777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457}
88242024-02-04T14:04:15.687Z{"msg":"Crucible daadbe13-4fd3-43db-aa56-829eb98688fd has session id: 29b68f65-fd11-4ec2-9e9d-1eefe68b3256","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717298911Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88252024-02-04T14:04:15.687Z{"msg":"Upstairs opts: Upstairs UUID: daadbe13-4fd3-43db-aa56-829eb98688fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717378567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88262024-02-04T14:04:15.687Z{"msg":"Crucible stats registered with UUID: daadbe13-4fd3-43db-aa56-829eb98688fd","v":0,"name":"crucible",Crucible runtime is spawned
88272024-02-04T14:04:15.688Z"level":30,"time":"2024-02-04T14:04:14.717539785Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88282024-02-04T14:04:15.688Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717618058Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457}
88292024-02-04T14:04:15.688Z{"msg":"daadbe13-4fd3-43db-aa56-829eb98688fd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717717539Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88302024-02-04T14:04:15.688Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.71778456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88312024-02-04T14:04:15.688Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.71785717Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88322024-02-04T14:04:15.688Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717889397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88332024-02-04T14:04:15.688Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717914175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88342024-02-04T14:04:15.688Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.717940664Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88352024-02-04T14:04:15.688Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.718096282Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88362024-02-04T14:04:15.688Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.718173183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"io task","client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88372024-02-04T14:04:15.688Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.718396915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"io task","client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88382024-02-04T14:04:15.688Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.71855509Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"io task","client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88392024-02-04T14:04:15.688Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.718586637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"io task","client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88402024-02-04T14:04:15.688Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.718610832Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"io task","client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88412024-02-04T14:04:15.688Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.71868667Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"io task","client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88422024-02-04T14:04:15.688Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.719533786Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88432024-02-04T14:04:15.688Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.719613772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88442024-02-04T14:04:15.688Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.719640635Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88452024-02-04T14:04:15.688Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.71966377Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88462024-02-04T14:04:15.688Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.719686264Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88472024-02-04T14:04:15.688Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.719769503Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88482024-02-04T14:04:15.693Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.720737899Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88492024-02-04T14:04:15.693Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.720804849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88502024-02-04T14:04:15.693Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.720930897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88512024-02-04T14:04:15.693Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.721770654Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88522024-02-04T14:04:15.693Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.721858989Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88532024-02-04T14:04:15.693Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.721885791Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88542024-02-04T14:04:15.693Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.721907152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88552024-02-04T14:04:15.693Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.721958559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88562024-02-04T14:04:15.693Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.721989266Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88572024-02-04T14:04:15.693Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722102907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88582024-02-04T14:04:15.693Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722154691Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88592024-02-04T14:04:15.693Z{"msg":"[0]R flush_numbers[0..12]: [127, 117, 120, 128, 128, 117, 123, 121, 124, 124, 127, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722181114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88602024-02-04T14:04:15.693Z{"msg":"[0]R generation[0..12]: [80, 74, 75, 81, 81, 74, 78, 76, 78, 78, 80, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722224751Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88612024-02-04T14:04:15.693Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722250559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88622024-02-04T14:04:15.693Z{"msg":"[1]R flush_numbers[0..12]: [127, 117, 120, 128, 128, 117, 123, 121, 124, 124, 127, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722274776Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88632024-02-04T14:04:15.693Z{"msg":"[1]R generation[0..12]: [80, 74, 75, 81, 81, 74, 78, 76, 78, 78, 80, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722316614Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88642024-02-04T14:04:15.693Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722342216Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88652024-02-04T14:04:15.694Z{"msg":"[2]R flush_numbers[0..12]: [127, 117, 120, 128, 128, 117, 123, 121, 124, 124, 127, 117]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
88662024-02-04T14:04:15.694Z,"time":"2024-02-04T14:04:14.722384351Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88672024-02-04T14:04:15.694Z{"msg":"[2]R generation[0..12]: [80, 74, 75, 81, 81, 74, 78, 76, 78, 78, 80, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722410862Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88682024-02-04T14:04:15.694Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722453991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88692024-02-04T14:04:15.694Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.72247738Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88702024-02-04T14:04:15.694Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722526016Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88712024-02-04T14:04:15.694Z{"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722549147Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88722024-02-04T14:04:15.694Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722590234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88732024-02-04T14:04:15.694Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722614146Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88742024-02-04T14:04:15.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722634555Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88752024-02-04T14:04:15.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722681454Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88762024-02-04T14:04:15.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722704791Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88772024-02-04T14:04:15.694Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722727396Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88782024-02-04T14:04:15.694Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.72277042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88792024-02-04T14:04:15.694Z{"msg":"daadbe13-4fd3-43db-aa56-829eb98688fd is now active with session: 29b68f65-fd11-4ec2-9e9d-1eefe68b3256","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722793703Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88802024-02-04T14:04:15.694Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722834794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457,"session_id":"29b68f65-fd11-4ec2-9e9d-1eefe68b3256"}
88812024-02-04T14:04:15.694Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:14.722856657Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457}
88822024-02-04T14:04:15.694ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88832024-02-04T14:04:15.694ZLoading write count information from file "/tmp/test_repair_verify.data"
88842024-02-04T14:04:15.694ZRun Repair workload
88852024-02-04T14:04:15.694Z01/30 Write block 255 len 16384 data: 10 9 8 8
88862024-02-04T14:04:15.697Z02/30 Read block 450 len 40960
88872024-02-04T14:04:15.698Z03/30 Read block 102 len 4096
88882024-02-04T14:04:15.698Z04/30 Write block 23 len 8192 data: 7 6
88892024-02-04T14:04:15.698Z05/30 Read block 534 len 20480
88902024-02-04T14:04:15.701Z06/30 Write block 74 len 40960 data: 8 8 8 9 8 8 7 8 8 6
88912024-02-04T14:04:15.701Z07/30 Read block 236 len 20480
88922024-02-04T14:04:15.703Z08/30 Write block 458 len 40960 data: 10 11 8 7 6 7 6 8 10 10
88932024-02-04T14:04:15.703Z09/30 Read block 307 len 24576
88942024-02-04T14:04:15.706Z10/30 Flush
88952024-02-04T14:04:15.710Z11/30 Write block 18 len 36864 data: 6 6 6 5 6 8 7 6 6
88962024-02-04T14:04:15.712Z12/30 Write block 261 len 20480 data: 8 9 9 8 7
88972024-02-04T14:04:15.715Z13/30 Write block 46 len 40960 data: 6 5 6 4 3 5 5 5 5 6
88982024-02-04T14:04:15.715Z14/30 Write block 248 len 12288 data: 9 12 13
88992024-02-04T14:04:15.715Z15/30 Read block 356 len 4096
89002024-02-04T14:04:15.717Z16/30 Read block 173 len 20480
89012024-02-04T14:04:15.718Z17/30 Read block 8 len 20480
89022024-02-04T14:04:15.720Z18/30 Flush
89032024-02-04T14:04:15.723Z19/30 Write block 451 len 28672 data: 8 10 9 11 10 11 11
89042024-02-04T14:04:15.726Z20/30 Write block 331 len 40960 data: 6 5 5 6 5 5 6 6 6 6
89052024-02-04T14:04:15.726Z21/30 Read block 461 len 20480
89062024-02-04T14:04:15.729Z22/30 Read block 223 len 28672
89072024-02-04T14:04:15.729Z23/30 Write block 542 len 32768 data: 5 7 5 5 3 3 4 5
89082024-02-04T14:04:15.732Z24/30 Write block 463 len 28672 data: 8 7 9 11 11 8 9
89092024-02-04T14:04:15.732Z25/30 Write block 163 len 24576 data: 7 6 7 8 8 8
89102024-02-04T14:04:15.734Z26/30 Write block 455 len 4096 data: 11
89112024-02-04T14:04:15.734Z27/30 Write block 234 len 36864 data: 7 4 4 6 6 5 6 6 5
89122024-02-04T14:04:15.737Z28/30 Read block 132 len 12288
89132024-02-04T14:04:15.738Z29/30 Read block 508 len 8192
89142024-02-04T14:04:15.743Z30/30 Write block 140 len 8192 data: 6 5
89152024-02-04T14:04:15.743Z----------------------------------------------------------------
89162024-02-04T14:04:15.744Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89172024-02-04T14:04:15.744ZStates: Active Active Active
89182024-02-04T14:04:15.744ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89192024-02-04T14:04:15.744Z 1 Acked 1000 Write 4 Done Done Done false
89202024-02-04T14:04:15.744Z 2 Acked 1001 Read 10 Done Done Done false
89212024-02-04T14:04:15.744Z 3 Acked 1002 Read 1 Done Done Done false
89222024-02-04T14:04:15.744Z 4 Acked 1003 Write 2 Done Done Sent false
89232024-02-04T14:04:15.744Z 5 Acked 1004 Read 5 Done Done Sent false
89242024-02-04T14:04:15.744Z 6 Acked 1005 Write 10 Done Done Sent false
89252024-02-04T14:04:15.744Z 7 Acked 1006 Read 5 Done Done Sent false
89262024-02-04T14:04:15.744Z 8 Acked 1007 Write 10 Done Done Sent false
89272024-02-04T14:04:15.744Z 9 Acked 1008 Read 6 Done Done Sent false
89282024-02-04T14:04:15.744Z 10 Acked 1009 Flush 0 Done Done Sent false
89292024-02-04T14:04:15.744Z 11 Acked 1010 Write 9 Done Done Sent false
89302024-02-04T14:04:15.744Z 12 Acked 1011 Write 5 Done Done Sent false
89312024-02-04T14:04:15.744Z 13 Acked 1012 Write 10 Done Done Sent false
89322024-02-04T14:04:15.744Z 14 Acked 1013 Write 3 Done Done Sent false
89332024-02-04T14:04:15.744Z 15 Acked 1014 Read 1 Done Done Sent false
89342024-02-04T14:04:15.744Z 16 Acked 1015 Read 5 Done Done Sent false
89352024-02-04T14:04:15.744Z 17 Acked 1016 Read 5 Done Done Sent false
89362024-02-04T14:04:15.744Z 18 Acked 1017 Flush 0 Done Done Sent false
89372024-02-04T14:04:15.744Z 19 Acked 1018 Write 7 Done Done Sent false
89382024-02-04T14:04:15.744Z 20 Acked 1019 Write 10 Done Done Sent false
89392024-02-04T14:04:15.744Z 21 Acked 1020 Read 5 Done Done Sent false
89402024-02-04T14:04:15.745Z 22 Acked 1021 Read 7 Done Done Sent false
89412024-02-04T14:04:15.745Z 23 Acked 1022 Write 8 Done Done Sent false
89422024-02-04T14:04:15.745Z 24 Acked 1023 Write 7 Done Done Sent false
89432024-02-04T14:04:15.745Z 25 Acked 1024 Write 6 Done Done Sent false
89442024-02-04T14:04:15.745Z 26 Acked 1025 Write 1 Done Done Sent false
89452024-02-04T14:04:15.745Z 27 Acked 1026 Write 9 Done Done Sent false
89462024-02-04T14:04:15.745Z 28 Acked 1027 Read 3 Done Done Sent false
89472024-02-04T14:04:15.745Z 29 Acked 1028 Read 2 Done Done Sent false
89482024-02-04T14:04:15.745Z 30 Acked 1029 Write 2 Sent Sent Sent false
89492024-02-04T14:04:15.745Z STATES DS:0 DS:1 DS:2 TOTAL
89502024-02-04T14:04:15.745Z New 0 0 0 0
89512024-02-04T14:04:15.745Z Sent 1 1 27 29
89522024-02-04T14:04:15.745Z Done 29 29 3 61
89532024-02-04T14:04:15.745Z Skipped 0 0 0 0
89542024-02-04T14:04:15.745Z Error 0 0 0 0
89552024-02-04T14:04:15.745ZLast Flush: 1017 1017 0
89562024-02-04T14:04:15.745ZDownstairs last five completed:
89572024-02-04T14:04:15.745ZUpstairs last five completed: 30 29 28 27 26
89582024-02-04T14:04:15.745Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:14.772650821Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1457}
89592024-02-04T14:04:15.745ZWrote out file "/tmp/test_repair_verify.data"
89602024-02-04T14:04:15.748Z
89612024-02-04T14:04:15.752ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89622024-02-04T14:04:15.780ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89632024-02-04T14:04:15.780Z 0 000-019 82 82 80 130 130 127 F F F <---
89642024-02-04T14:04:15.780Z 1 020-039 82 82 74 130 130 117 F F F <---
89652024-02-04T14:04:15.780Z 2 040-059 82 82 75 130 130 120 F F F <---
89662024-02-04T14:04:15.780Z 3 060-079 82 82 81 129 129 128 F F F <---
89672024-02-04T14:04:15.780Z 4 080-099 82 82 81 129 129 128 F F F <---
89682024-02-04T14:04:15.780Z 5 100-119 74 74 74 117 117 117 F F F
89692024-02-04T14:04:15.780Z 6 120-139 78 78 78 123 123 123 F F F
89702024-02-04T14:04:15.785Z 7 140-159 76 76 76 121 121 121 T T F <---
89712024-02-04T14:04:15.785Z 8 160-179 78 78 78 124 124 124 T T F <---
89722024-02-04T14:04:15.785Z 9 180-199 78 78 78 124 124 124 F F F
89732024-02-04T14:04:15.785Z 10 200-219 80 80 80 127 127 127 F F F
89742024-02-04T14:04:15.785Z 11 220-239 74 74 74 117 117 117 T T F <---
89752024-02-04T14:04:15.785Z 12 240-259 82 82 81 130 130 128 T T T <---
89762024-02-04T14:04:15.785Z 13 260-279 82 82 73 130 130 116 F F F <---
89772024-02-04T14:04:15.785Z 14 280-299 80 80 80 127 127 127 F F F
89782024-02-04T14:04:15.785Z 15 300-319 74 74 74 119 119 119 F F F
89792024-02-04T14:04:15.785Z 16 320-339 78 78 78 124 124 124 T T F <---
89802024-02-04T14:04:15.785Z 17 340-359 78 78 78 124 124 124 T T F <---
89812024-02-04T14:04:15.785Z 18 360-379 80 80 80 127 127 127 F F F
89822024-02-04T14:04:15.785Z 19 380-399 79 79 79 126 126 126 F F F
89832024-02-04T14:04:15.785Z 20 400-419 81 81 81 128 128 128 F F F
89842024-02-04T14:04:15.785Z 21 420-439 77 77 77 122 122 122 F F F
89852024-02-04T14:04:15.785Z 22 440-459 82 82 81 129 129 128 T T F <---
89862024-02-04T14:04:15.785Z 23 460-479 82 82 81 129 129 128 T T F <---
89872024-02-04T14:04:15.785Z 24 480-499 79 79 79 126 126 126 F F F
89882024-02-04T14:04:15.786Z 25 500-519 77 77 77 122 122 122 F F F
89892024-02-04T14:04:15.786Z 26 520-539 77 77 77 122 122 122 F F F
89902024-02-04T14:04:15.786Z 27 540-559 77 77 77 122 122 122 T T F <---
89912024-02-04T14:04:15.786Z 28 560-579 81 81 81 128 128 128 F F F
89922024-02-04T14:04:15.786Z 29 580-599 81 81 81 128 128 128 F F F
89932024-02-04T14:04:15.786ZMax gen: 82, Max flush: 130
89942024-02-04T14:04:15.786ZError: Difference in extent metadata found!
89952024-02-04T14:04:15.786ZA Difference in extent metadata is expected here
89962024-02-04T14:04:15.786ZOn loop 41
89972024-02-04T14:04:15.786Z
89982024-02-04T14:04:15.789ZVerifying data now
89992024-02-04T14:04:21.067ZNo change after verify
90002024-02-04T14:04:21.067ZLoop: 41 Downstairs dump after verify (and repair):
90012024-02-04T14:04:21.090ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90022024-02-04T14:04:21.090Z 0 000-019 82 82 82 130 130 130 F F F
90032024-02-04T14:04:21.091Z 1 020-039 82 82 82 130 130 130 F F F
90042024-02-04T14:04:21.091Z 2 040-059 82 82 82 130 130 130 F F F
90052024-02-04T14:04:21.091Z 3 060-079 82 82 82 129 129 129 F F F
90062024-02-04T14:04:21.091Z 4 080-099 82 82 82 129 129 129 F F F
90072024-02-04T14:04:21.091Z 5 100-119 74 74 74 117 117 117 F F F
90082024-02-04T14:04:21.091Z 6 120-139 78 78 78 123 123 123 F F F
90092024-02-04T14:04:21.091Z 7 140-159 83 83 83 131 131 131 F F F
90102024-02-04T14:04:21.091Z 8 160-179 83 83 83 131 131 131 F F F
90112024-02-04T14:04:21.091Z 9 180-199 78 78 78 124 124 124 F F F
90122024-02-04T14:04:21.091Z 10 200-219 80 80 80 127 127 127 F F F
90132024-02-04T14:04:21.095Z 11 220-239 83 83 83 131 131 131 F F F
90142024-02-04T14:04:21.095Z 12 240-259 83 83 83 131 131 131 F F F
90152024-02-04T14:04:21.095Z 13 260-279 82 82 82 130 130 130 F F F
90162024-02-04T14:04:21.095Z 14 280-299 80 80 80 127 127 127 F F F
90172024-02-04T14:04:21.095Z 15 300-319 74 74 74 119 119 119 F F F
90182024-02-04T14:04:21.096Z 16 320-339 83 83 83 131 131 131 F F F
90192024-02-04T14:04:21.096Z 17 340-359 83 83 83 131 131 131 F F F
90202024-02-04T14:04:21.096Z 18 360-379 80 80 80 127 127 127 F F F
90212024-02-04T14:04:21.096Z 19 380-399 79 79 79 126 126 126 F F F
90222024-02-04T14:04:21.096Z 20 400-419 81 81 81 128 128 128 F F F
90232024-02-04T14:04:21.096Z 21 420-439 77 77 77 122 122 122 F F F
90242024-02-04T14:04:21.096Z 22 440-459 83 83 83 131 131 131 F F F
90252024-02-04T14:04:21.096Z 23 460-479 83 83 83 131 131 131 F F F
90262024-02-04T14:04:21.096Z 24 480-499 79 79 79 126 126 126 F F F
90272024-02-04T14:04:21.096Z 25 500-519 77 77 77 122 122 122 F F F
90282024-02-04T14:04:21.096Z 26 520-539 77 77 77 122 122 122 F F F
90292024-02-04T14:04:21.096Z 27 540-559 83 83 83 131 131 131 F F F
90302024-02-04T14:04:21.096Z 28 560-579 81 81 81 128 128 128 F F F
90312024-02-04T14:04:21.097Z 29 580-599 81 81 81 128 128 128 F F F
90322024-02-04T14:04:21.097ZMax gen: 83, Max flush: 131
90332024-02-04T14:04:21.097Z
90342024-02-04T14:04:21.097ZBegin loop 42
90352024-02-04T14:04:21.097ZDownstairs to restart: 1
90362024-02-04T14:04:21.125Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.158106881Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465}
90372024-02-04T14:04:21.125Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.15858477Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465}
90382024-02-04T14:04:21.130Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.15863994Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465}
90392024-02-04T14:04:21.130Z{"msg":"Crucible 77b460d2-e8fc-47ae-b338-f129005e9a4b has session id: 065560a3-3fd4-407b-abd2-e708511ef539","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.158801655Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90402024-02-04T14:04:21.130Z{"msg":"Crucible runtime is spawned
90412024-02-04T14:04:21.130ZUpstairs opts: Upstairs UUID: 77b460d2-e8fc-47ae-b338-f129005e9a4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.158899754Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90422024-02-04T14:04:21.130Z{"msg":"Crucible stats registered with UUID: 77b460d2-e8fc-47ae-b338-f129005e9a4b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.158991721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90432024-02-04T14:04:21.130Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159038474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465}
90442024-02-04T14:04:21.131Z{"msg":"77b460d2-e8fc-47ae-b338-f129005e9a4b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159129541Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90452024-02-04T14:04:21.131Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159192676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90462024-02-04T14:04:21.131Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159384074Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90472024-02-04T14:04:21.131Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159415056Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90482024-02-04T14:04:21.131Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159447756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90492024-02-04T14:04:21.131Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159517178Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90502024-02-04T14:04:21.131Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159560536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90512024-02-04T14:04:21.131Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159679897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90522024-02-04T14:04:21.131Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159801767Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90532024-02-04T14:04:21.131Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159850098Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90542024-02-04T14:04:21.131Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159896188Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90552024-02-04T14:04:21.131Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.159972456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90562024-02-04T14:04:21.131Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.160015954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90572024-02-04T14:04:21.131Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.160907358Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90582024-02-04T14:04:21.131Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.160994448Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90592024-02-04T14:04:21.132Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.161036983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90602024-02-04T14:04:21.132Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.161081391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90612024-02-04T14:04:21.132Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.161424005Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90622024-02-04T14:04:21.132Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.161505753Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90632024-02-04T14:04:21.132Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.162225723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90642024-02-04T14:04:21.132Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.162383323Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90652024-02-04T14:04:21.132Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.162434818Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90662024-02-04T14:04:21.136Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163212296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90672024-02-04T14:04:21.136Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163300584Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90682024-02-04T14:04:21.136Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163372373Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90692024-02-04T14:04:21.137Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163442291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90702024-02-04T14:04:21.137Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163514424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90712024-02-04T14:04:21.137Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163559907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90722024-02-04T14:04:21.137Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163658177Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90732024-02-04T14:04:21.137Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163684444Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90742024-02-04T14:04:21.137Z{"msg":"[0]R flush_numbers[0..12]: [130, 130, 130, 129, 129, 117, 123, 131, 131, 124, 127, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163727621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90752024-02-04T14:04:21.137Z{"msg":"[0]R generation[0..12]: [82, 82, 82, 82, 82, 74, 78, 83, 83, 78, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163752223Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90762024-02-04T14:04:21.137Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163776151Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90772024-02-04T14:04:21.137Z{"msg":"[1]R flush_numbers[0..12]: [130, 130, 130, 129, 129, 117, 123, 131, 131, 124, 127, 131]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
90782024-02-04T14:04:21.137Z,"time":"2024-02-04T14:04:20.16383513Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90792024-02-04T14:04:21.137Z{"msg":"[1]R generation[0..12]: [82, 82, 82, 82, 82, 74, 78, 83, 83, 78, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163881984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90802024-02-04T14:04:21.137Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163907406Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90812024-02-04T14:04:21.137Z{"msg":"[2]R flush_numbers[0..12]: [130, 130, 130, 129, 129, 117, 123, 131, 131, 124, 127, 131]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.163956936Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90822024-02-04T14:04:21.137Z{"msg":"[2]R generation[0..12]: [82, 82, 82, 82, 82, 74, 78, 83, 83, 78, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164042659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90832024-02-04T14:04:21.137Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164089716Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90842024-02-04T14:04:21.137Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164113651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90852024-02-04T14:04:21.137Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164154222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90862024-02-04T14:04:21.137Z{"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164177221Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90872024-02-04T14:04:21.137Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164217526Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90882024-02-04T14:04:21.137Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164241224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90892024-02-04T14:04:21.138Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164280763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90902024-02-04T14:04:21.138Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164303925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90912024-02-04T14:04:21.138Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164325884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90922024-02-04T14:04:21.138Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164367428Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90932024-02-04T14:04:21.138Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164389565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90942024-02-04T14:04:21.138Z{"msg":"77b460d2-e8fc-47ae-b338-f129005e9a4b is now active with session: 065560a3-3fd4-407b-abd2-e708511ef539","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164411431Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90952024-02-04T14:04:21.138Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164431412Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465,"session_id":"065560a3-3fd4-407b-abd2-e708511ef539"}
90962024-02-04T14:04:21.138Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:20.164470099Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465}
90972024-02-04T14:04:21.138ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90982024-02-04T14:04:21.138ZLoading write count information from file "/tmp/test_repair_verify.data"
90992024-02-04T14:04:21.138ZRun Repair workload
91002024-02-04T14:04:21.138Z01/30 Write block 575 len 20480 data: 8 11 11 10 12
91012024-02-04T14:04:21.138Z02/30 Read block 159 len 8192
91022024-02-04T14:04:21.141Z03/30 Write block 9 len 32768 data: 8 5 4 3 3 4 6 6
91032024-02-04T14:04:21.141Z04/30 Read block 292 len 20480
91042024-02-04T14:04:21.142Z05/30 Read block 260 len 8192
91052024-02-04T14:04:21.142Z06/30 Write block 237 len 32768 data: 7 7 6 7 7 6 5 5
91062024-02-04T14:04:21.142Z07/30 Write block 527 len 8192 data: 5 5
91072024-02-04T14:04:21.146Z08/30 Read block 141 len 20480
91082024-02-04T14:04:21.146Z09/30 Write block 117 len 32768 data: 9 6 6 6 6 6 6 5
91092024-02-04T14:04:21.147Z10/30 Write block 101 len 16384 data: 5 4 4 6
91102024-02-04T14:04:21.147Z11/30 Read block 451 len 4096
91112024-02-04T14:04:21.162Z12/30 Write block 366 len 12288 data: 6 8 7
91122024-02-04T14:04:21.162Z13/30 Read block 430 len 12288
91132024-02-04T14:04:21.162Z14/30 Read block 124 len 20480
91142024-02-04T14:04:21.165Z15/30 Read block 530 len 20480
91152024-02-04T14:04:21.165Z16/30 Write block 225 len 28672 data: 6 6 7 7 6 5 4
91162024-02-04T14:04:21.165Z17/30 Write block 265 len 28672 data: 8 8 7 7 7 6 7
91172024-02-04T14:04:21.165Z18/30 Read block 332 len 12288
91182024-02-04T14:04:21.165Z19/30 Write block 563 len 24576 data: 8 8 9 9 7 5
91192024-02-04T14:04:21.165Z20/30 Flush
91202024-02-04T14:04:21.165Z21/30 Write block 21 len 20480 data: 6 7 9 8 7
91212024-02-04T14:04:21.165Z22/30 Flush
91222024-02-04T14:04:21.169Z23/30 Write block 117 len 36864 data: 10 7 7 7 7 7 7 6 8
91232024-02-04T14:04:21.169Z24/30 Write block 80 len 28672 data: 8 9 9 7 7 8 7
91242024-02-04T14:04:21.169Z25/30 Read block 309 len 4096
91252024-02-04T14:04:21.172Z26/30 Flush
91262024-02-04T14:04:21.177Z27/30 Write block 468 len 40960 data: 9 10 9 6 7 7 7 9 7 6
91272024-02-04T14:04:21.178Z28/30 Read block 121 len 36864
91282024-02-04T14:04:21.181Z29/30 Read block 223 len 8192
91292024-02-04T14:04:21.181Z30/30 Write block 581 len 36864 data: 11 7 9 11 10 9 8 9 6
91302024-02-04T14:04:21.182Z----------------------------------------------------------------
91312024-02-04T14:04:21.182Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91322024-02-04T14:04:21.182ZStates: Active Active Active
91332024-02-04T14:04:21.182ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91342024-02-04T14:04:21.182Z 1 Acked 1000 Write 5 Done Sent Done false
91352024-02-04T14:04:21.182Z 2 Acked 1001 Read 2 Done Sent Done false
91362024-02-04T14:04:21.182Z 3 Acked 1002 Write 8 Done Sent Done false
91372024-02-04T14:04:21.182Z 4 Acked 1003 Read 5 Done Sent Done false
91382024-02-04T14:04:21.182Z 5 Acked 1004 Read 2 Done Sent Done false
91392024-02-04T14:04:21.182Z 6 Acked 1005 Write 8 Done Sent Done false
91402024-02-04T14:04:21.182Z 7 Acked 1006 Write 2 Done Sent Done false
91412024-02-04T14:04:21.182Z 8 Acked 1007 Read 5 Done Sent Done false
91422024-02-04T14:04:21.182Z 9 Acked 1008 Write 8 Done Sent Done false
91432024-02-04T14:04:21.182Z 10 Acked 1009 Write 4 Done Sent Done false
91442024-02-04T14:04:21.182Z 11 Acked 1010 Read 1 Done Sent Done false
91452024-02-04T14:04:21.182Z 12 Acked 1011 Write 3 Done Sent Done false
91462024-02-04T14:04:21.182Z 13 Acked 1012 Read 3 Done Sent Done false
91472024-02-04T14:04:21.182Z 14 Acked 1013 Read 5 Done Sent Done false
91482024-02-04T14:04:21.182Z 15 Acked 1014 Read 5 Done Sent Done false
91492024-02-04T14:04:21.182Z 16 Acked 1015 Write 7 Done Sent Done false
91502024-02-04T14:04:21.182Z 17 Acked 1016 Write 7 Done Sent Done false
91512024-02-04T14:04:21.182Z 18 Acked 1017 Read 3 Done Sent Done false
91522024-02-04T14:04:21.182Z 19 Acked 1018 Write 6 Done Sent Done false
91532024-02-04T14:04:21.182Z 20 Acked 1019 Flush 0 Done Sent Done false
91542024-02-04T14:04:21.183Z 21 Acked 1020 Write 5 Done Sent Done false
91552024-02-04T14:04:21.183Z 22 Acked 1021 Flush 0 Done Sent Done false
91562024-02-04T14:04:21.183Z 23 Acked 1022 Write 9 Done Sent Done false
91572024-02-04T14:04:21.183Z 24 Acked 1023 Write 7 Done Sent Done false
91582024-02-04T14:04:21.183Z 25 Acked 1024 Read 1 Done Sent Done false
91592024-02-04T14:04:21.183Z 26 Acked 1025 Flush 0 Done Sent Done false
91602024-02-04T14:04:21.183Z 27 Acked 1026 Write 10 Done Sent Done false
91612024-02-04T14:04:21.183Z 28 Acked 1027 Read 9 Done Sent Done false
91622024-02-04T14:04:21.183Z 29 Acked 1028 Read 2 Done Sent Done false
91632024-02-04T14:04:21.183Z 30 Acked 1029 Write 9 Sent Sent Sent false
91642024-02-04T14:04:21.183Z STATES DS:0 DS:1 DS:2 TOTAL
91652024-02-04T14:04:21.183Z New 0 0 0 0
91662024-02-04T14:04:21.183Z Sent 1 30 1 32
91672024-02-04T14:04:21.183Z Done 29 0 29 58
91682024-02-04T14:04:21.183Z Skipped 0 0 0 0
91692024-02-04T14:04:21.183Z Error 0 0 0 0
91702024-02-04T14:04:21.183ZLast Flush: 1025 0 1025
91712024-02-04T14:04:21.183ZDownstairs last five completed:
91722024-02-04T14:04:21.183ZUpstairs last five completed: 30 29 28 27 26
91732024-02-04T14:04:21.183Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:20.214003498Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1465}
91742024-02-04T14:04:21.187ZWrote out file "/tmp/test_repair_verify.data"
91752024-02-04T14:04:21.190Z
91762024-02-04T14:04:21.196ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91772024-02-04T14:04:21.217ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91782024-02-04T14:04:21.217Z 0 000-019 84 82 84 132 130 132 F F F <---
91792024-02-04T14:04:21.217Z 1 020-039 84 82 84 133 130 133 F F F <---
91802024-02-04T14:04:21.218Z 2 040-059 82 82 82 130 130 130 F F F
91812024-02-04T14:04:21.218Z 3 060-079 82 82 82 129 129 129 F F F
91822024-02-04T14:04:21.218Z 4 080-099 84 82 84 134 129 134 F F F <---
91832024-02-04T14:04:21.221Z 5 100-119 84 74 84 134 117 134 F F F <---
91842024-02-04T14:04:21.221Z 6 120-139 84 78 84 134 123 134 F F F <---
91852024-02-04T14:04:21.221Z 7 140-159 83 83 83 131 131 131 F F F
91862024-02-04T14:04:21.222Z 8 160-179 83 83 83 131 131 131 F F F
91872024-02-04T14:04:21.222Z 9 180-199 78 78 78 124 124 124 F F F
91882024-02-04T14:04:21.222Z 10 200-219 80 80 80 127 127 127 F F F
91892024-02-04T14:04:21.222Z 11 220-239 84 83 84 132 131 132 F F F <---
91902024-02-04T14:04:21.222Z 12 240-259 84 83 84 132 131 132 F F F <---
91912024-02-04T14:04:21.222Z 13 260-279 84 82 84 132 130 132 F F F <---
91922024-02-04T14:04:21.222Z 14 280-299 80 80 80 127 127 127 F F F
91932024-02-04T14:04:21.222Z 15 300-319 74 74 74 119 119 119 F F F
91942024-02-04T14:04:21.222Z 16 320-339 83 83 83 131 131 131 F F F
91952024-02-04T14:04:21.222Z 17 340-359 83 83 83 131 131 131 F F F
91962024-02-04T14:04:21.222Z 18 360-379 84 80 84 132 127 132 F F F <---
91972024-02-04T14:04:21.222Z 19 380-399 79 79 79 126 126 126 F F F
91982024-02-04T14:04:21.222Z 20 400-419 81 81 81 128 128 128 F F F
91992024-02-04T14:04:21.222Z 21 420-439 77 77 77 122 122 122 F F F
92002024-02-04T14:04:21.222Z 22 440-459 83 83 83 131 131 131 F F F
92012024-02-04T14:04:21.222Z 23 460-479 83 83 83 131 131 131 T F T <---
92022024-02-04T14:04:21.222Z 24 480-499 79 79 79 126 126 126 F F F
92032024-02-04T14:04:21.222Z 25 500-519 77 77 77 122 122 122 F F F
92042024-02-04T14:04:21.222Z 26 520-539 84 77 84 132 122 132 F F F <---
92052024-02-04T14:04:21.222Z 27 540-559 83 83 83 131 131 131 F F F
92062024-02-04T14:04:21.222Z 28 560-579 84 81 84 132 128 132 F F F <---
92072024-02-04T14:04:21.222Z 29 580-599 81 81 81 128 128 128 T F T <---
92082024-02-04T14:04:21.222ZMax gen: 84, Max flush: 134
92092024-02-04T14:04:21.222ZError: Difference in extent metadata found!
92102024-02-04T14:04:21.223ZA Difference in extent metadata is expected here
92112024-02-04T14:04:21.223ZOn loop 42
92122024-02-04T14:04:21.223Z
92132024-02-04T14:04:21.226ZVerifying data now
92142024-02-04T14:04:26.338ZNo change after verify
92152024-02-04T14:04:26.338ZLoop: 42 Downstairs dump after verify (and repair):
92162024-02-04T14:04:26.358ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92172024-02-04T14:04:26.358Z 0 000-019 84 84 84 132 132 132 F F F
92182024-02-04T14:04:26.358Z 1 020-039 84 84 84 133 133 133 F F F
92192024-02-04T14:04:26.358Z 2 040-059 82 82 82 130 130 130 F F F
92202024-02-04T14:04:26.358Z 3 060-079 82 82 82 129 129 129 F F F
92212024-02-04T14:04:26.358Z 4 080-099 84 84 84 134 134 134 F F F
92222024-02-04T14:04:26.361Z 5 100-119 84 84 84 134 134 134 F F F
92232024-02-04T14:04:26.361Z 6 120-139 84 84 84 134 134 134 F F F
92242024-02-04T14:04:26.361Z 7 140-159 83 83 83 131 131 131 F F F
92252024-02-04T14:04:26.361Z 8 160-179 83 83 83 131 131 131 F F F
92262024-02-04T14:04:26.361Z 9 180-199 78 78 78 124 124 124 F F F
92272024-02-04T14:04:26.361Z 10 200-219 80 80 80 127 127 127 F F F
92282024-02-04T14:04:26.361Z 11 220-239 84 84 84 132 132 132 F F F
92292024-02-04T14:04:26.361Z 12 240-259 84 84 84 132 132 132 F F F
92302024-02-04T14:04:26.361Z 13 260-279 84 84 84 132 132 132 F F F
92312024-02-04T14:04:26.361Z 14 280-299 80 80 80 127 127 127 F F F
92322024-02-04T14:04:26.361Z 15 300-319 74 74 74 119 119 119 F F F
92332024-02-04T14:04:26.361Z 16 320-339 83 83 83 131 131 131 F F F
92342024-02-04T14:04:26.361Z 17 340-359 83 83 83 131 131 131 F F F
92352024-02-04T14:04:26.361Z 18 360-379 84 84 84 132 132 132 F F F
92362024-02-04T14:04:26.362Z 19 380-399 79 79 79 126 126 126 F F F
92372024-02-04T14:04:26.362Z 20 400-419 81 81 81 128 128 128 F F F
92382024-02-04T14:04:26.362Z 21 420-439 77 77 77 122 122 122 F F F
92392024-02-04T14:04:26.362Z 22 440-459 83 83 83 131 131 131 F F F
92402024-02-04T14:04:26.362Z 23 460-479 85 85 85 135 135 135 F F F
92412024-02-04T14:04:26.362Z 24 480-499 79 79 79 126 126 126 F F F
92422024-02-04T14:04:26.362Z 25 500-519 77 77 77 122 122 122 F F F
92432024-02-04T14:04:26.362Z 26 520-539 84 84 84 132 132 132 F F F
92442024-02-04T14:04:26.362Z 27 540-559 83 83 83 131 131 131 F F F
92452024-02-04T14:04:26.362Z 28 560-579 84 84 84 132 132 132 F F F
92462024-02-04T14:04:26.362Z 29 580-599 85 85 85 135 135 135 F F F
92472024-02-04T14:04:26.362ZMax gen: 85, Max flush: 135
92482024-02-04T14:04:26.365Z
92492024-02-04T14:04:26.365ZBegin loop 43
92502024-02-04T14:04:26.365ZDownstairs to restart: 1
92512024-02-04T14:04:26.392Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.425054575Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473}
92522024-02-04T14:04:26.392Z{Crucible runtime is spawned
92532024-02-04T14:04:26.396Z"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.425733618Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473}
92542024-02-04T14:04:26.396Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.425812714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473}
92552024-02-04T14:04:26.396Z{"msg":"Crucible c89a3d66-57a7-42ba-84b7-4824acf97681 has session id: 70dd4c58-5c20-41b1-b011-ac77be21f479","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.42585306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92562024-02-04T14:04:26.396Z{"msg":"Upstairs opts: Upstairs UUID: c89a3d66-57a7-42ba-84b7-4824acf97681, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.42593762Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92572024-02-04T14:04:26.396Z{"msg":"Crucible stats registered with UUID: c89a3d66-57a7-42ba-84b7-4824acf97681","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426117487Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92582024-02-04T14:04:26.396Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426163222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473}
92592024-02-04T14:04:26.396Z{"msg":"c89a3d66-57a7-42ba-84b7-4824acf97681 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426230884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92602024-02-04T14:04:26.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426267036Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92612024-02-04T14:04:26.396Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426361337Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92622024-02-04T14:04:26.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426404888Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92632024-02-04T14:04:26.396Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426470214Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92642024-02-04T14:04:26.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426537041Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92652024-02-04T14:04:26.396Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426585229Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92662024-02-04T14:04:26.396Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426664695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"io task","client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92672024-02-04T14:04:26.396Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426709841Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"io task","client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92682024-02-04T14:04:26.397Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426754517Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"io task","client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92692024-02-04T14:04:26.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426813394Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"io task","client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92702024-02-04T14:04:26.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426857524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"io task","client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92712024-02-04T14:04:26.397Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.426898377Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"io task","client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92722024-02-04T14:04:26.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.427786832Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92732024-02-04T14:04:26.397Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.427833178Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92742024-02-04T14:04:26.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.427927197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92752024-02-04T14:04:26.397Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.428022697Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92762024-02-04T14:04:26.397Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.428578702Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92772024-02-04T14:04:26.401Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.42884804Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92782024-02-04T14:04:26.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.429147391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92792024-02-04T14:04:26.401Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.429235302Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92802024-02-04T14:04:26.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.42935204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92812024-02-04T14:04:26.402Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.429418367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92822024-02-04T14:04:26.402Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.429445728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92832024-02-04T14:04:26.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.429684665Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92842024-02-04T14:04:26.402Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.429745177Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92852024-02-04T14:04:26.402Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.429770498Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92862024-02-04T14:04:26.402Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.430093162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92872024-02-04T14:04:26.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.430773076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92882024-02-04T14:04:26.402Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.430849414Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92892024-02-04T14:04:26.402Z{"msg":"[0]R flush_numbers[0..12]: [132, 133, 130, 129, 134, 134, 134, 131, 131, 124, 127, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.430882893Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92902024-02-04T14:04:26.402Z{"msg":"[0]R generation[0..12]: [84, 84, 82, 82, 84, 84, 84, 83, 83, 78, 80, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.430908322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92912024-02-04T14:04:26.402Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.430932409Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92922024-02-04T14:04:26.402Z{"msg":"[1]R flush_numbers[0..12]: [132, 133, 130, 129, 134, 134, 134, 131, 131, 124, 127, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.430979394Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92932024-02-04T14:04:26.402Z{"msg":"[1]R generation[0..12]: [84, 84, 82, 82, 84, 84, 84, 83, 83, 78, 80, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431004004Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92942024-02-04T14:04:26.402Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431049439Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92952024-02-04T14:04:26.402Z{"msg":"[2]R flush_numbers[0..12]: [132, 133, 130, 129, 134, 134, 134, 131, 131, 124, 127, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431073932Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92962024-02-04T14:04:26.402Z{"msg":"[2]R generation[0..12]: [84, 84, 82, 82, 84, 84, 84, 83, 83, 78, 80, 84]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
92972024-02-04T14:04:26.402Z,"time":"2024-02-04T14:04:25.431098528Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92982024-02-04T14:04:26.402Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431126254Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
92992024-02-04T14:04:26.402Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.43114814Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93002024-02-04T14:04:26.402Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431170479Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93012024-02-04T14:04:26.402Z{"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431193231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93022024-02-04T14:04:26.402Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431215012Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93032024-02-04T14:04:26.402Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431239084Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93042024-02-04T14:04:26.403Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431259334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93052024-02-04T14:04:26.403Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.4312809Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93062024-02-04T14:04:26.403Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431302795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93072024-02-04T14:04:26.403Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431325124Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93082024-02-04T14:04:26.403Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431370038Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93092024-02-04T14:04:26.403Z{"msg":"c89a3d66-57a7-42ba-84b7-4824acf97681 is now active with session: 70dd4c58-5c20-41b1-b011-ac77be21f479","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431395117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93102024-02-04T14:04:26.403Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.43141547Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473,"session_id":"70dd4c58-5c20-41b1-b011-ac77be21f479"}
93112024-02-04T14:04:26.403Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:25.431435715Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473}
93122024-02-04T14:04:26.403ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93132024-02-04T14:04:26.403ZLoading write count information from file "/tmp/test_repair_verify.data"
93142024-02-04T14:04:26.403ZRun Repair workload
93152024-02-04T14:04:26.403Z01/30 Write block 184 len 28672 data: 9 10 8 7 6 6 4
93162024-02-04T14:04:26.406Z02/30 Read block 270 len 16384
93172024-02-04T14:04:26.406Z03/30 Flush
93182024-02-04T14:04:26.409Z04/30 Write block 18 len 4096 data: 7
93192024-02-04T14:04:26.409Z05/30 Read block 440 len 32768
93202024-02-04T14:04:26.412Z06/30 Write block 324 len 20480 data: 5 5 5 5 4
93212024-02-04T14:04:26.412Z07/30 Read block 294 len 20480
93222024-02-04T14:04:26.412Z08/30 Flush
93232024-02-04T14:04:26.415Z09/30 Write block 82 len 16384 data: 10 8 8 9
93242024-02-04T14:04:26.418Z10/30 Read block 174 len 16384
93252024-02-04T14:04:26.418Z11/30 Read block 582 len 12288
93262024-02-04T14:04:26.421Z12/30 Write block 438 len 20480 data: 5 4 5 6 6
93272024-02-04T14:04:26.421Z13/30 Read block 398 len 24576
93282024-02-04T14:04:26.564Z14/30 Read block 396 len 40960
93292024-02-04T14:04:26.564Z15/30 Read block 76 len 24576
93302024-02-04T14:04:26.568Z16/30 Read block 543 len 40960
93312024-02-04T14:04:26.568Z17/30 Flush
93322024-02-04T14:04:26.568Z18/30 Write block 268 len 4096 data: 8
93332024-02-04T14:04:26.568Z19/30 Read block 554 len 4096
93342024-02-04T14:04:26.568Z20/30 Read block 129 len 16384
93352024-02-04T14:04:26.568Z21/30 Write block 590 len 24576 data: 7 6 7 7 8 8
93362024-02-04T14:04:26.568Z22/30 Read block 287 len 40960
93372024-02-04T14:04:26.568Z23/30 Read block 251 len 32768
93382024-02-04T14:04:26.568Z24/30 Read block 77 len 32768
93392024-02-04T14:04:26.568Z25/30 Read block 580 len 4096
93402024-02-04T14:04:26.568Z26/30 Write block 490 len 32768 data: 8 9 11 11 10 9 10 9
93412024-02-04T14:04:26.569Z27/30 Read block 397 len 16384
93422024-02-04T14:04:26.569Z28/30 Read block 399 len 24576
93432024-02-04T14:04:26.569Z29/30 Write block 4 len 4096 data: 7
93442024-02-04T14:04:26.569Z30/30 Write block 409 len 16384 data: 7 7 9 8
93452024-02-04T14:04:26.569Z----------------------------------------------------------------
93462024-02-04T14:04:26.569Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
93472024-02-04T14:04:26.569ZStates: Active Active Active
93482024-02-04T14:04:26.569ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93492024-02-04T14:04:26.569Z 1 Acked 1000 Write 7 Done Done Done false
93502024-02-04T14:04:26.569Z 2 Acked 1001 Read 4 Done Done Done false
93512024-02-04T14:04:26.569Z 3 Acked 1002 Flush 0 Done Sent Done false
93522024-02-04T14:04:26.569Z 4 Acked 1003 Write 1 Done Sent Done false
93532024-02-04T14:04:26.569Z 5 Acked 1004 Read 8 Done Sent Done false
93542024-02-04T14:04:26.569Z 6 Acked 1005 Write 5 Done Sent Done false
93552024-02-04T14:04:26.569Z 7 Acked 1006 Read 5 Done Sent Done false
93562024-02-04T14:04:26.569Z 8 Acked 1007 Flush 0 Done Sent Done false
93572024-02-04T14:04:26.569Z 9 Acked 1008 Write 4 Done Sent Done false
93582024-02-04T14:04:26.569Z 10 Acked 1009 Read 4 Done Sent Done false
93592024-02-04T14:04:26.569Z 11 Acked 1010 Read 3 Done Sent Done false
93602024-02-04T14:04:26.569Z 12 Acked 1011 Write 5 Done Sent Done false
93612024-02-04T14:04:26.569Z 13 Acked 1012 Read 6 Done Sent Done false
93622024-02-04T14:04:26.569Z 14 Acked 1013 Read 10 Done Sent Done false
93632024-02-04T14:04:26.569Z 15 Acked 1014 Read 6 Done Sent Done false
93642024-02-04T14:04:26.569Z 16 Acked 1015 Read 10 Done Sent Done false
93652024-02-04T14:04:26.569Z 17 Acked 1016 Flush 0 Done Sent Done false
93662024-02-04T14:04:26.570Z 18 Acked 1017 Write 1 Done Sent Done false
93672024-02-04T14:04:26.570Z 19 Acked 1018 Read 1 Done Sent Done false
93682024-02-04T14:04:26.570Z 20 Acked 1019 Read 4 Done Sent Done false
93692024-02-04T14:04:26.570Z 21 Acked 1020 Write 6 Done Sent Done false
93702024-02-04T14:04:26.570Z 22 Acked 1021 Read 10 Done Sent Done false
93712024-02-04T14:04:26.570Z 23 Acked 1022 Read 8 Done Sent Done false
93722024-02-04T14:04:26.570Z 24 Acked 1023 Read 8 Done Sent Done false
93732024-02-04T14:04:26.570Z 25 Acked 1024 Read 1 Done Sent Done false
93742024-02-04T14:04:26.570Z 26 Acked 1025 Write 8 Done Sent Done false
93752024-02-04T14:04:26.570Z 27 Acked 1026 Read 4 Done Sent Done false
93762024-02-04T14:04:26.570Z 28 Acked 1027 Read 6 Done Sent Done false
93772024-02-04T14:04:26.570Z 29 Acked 1028 Write 1 Done Sent Done false
93782024-02-04T14:04:26.570Z 30 Acked 1029 Write 4 Sent Sent Sent false
93792024-02-04T14:04:26.570Z STATES DS:0 DS:1 DS:2 TOTAL
93802024-02-04T14:04:26.570Z New 0 0 0 0
93812024-02-04T14:04:26.570Z Sent 1 28 1 30
93822024-02-04T14:04:26.570Z Done 29 2 29 60
93832024-02-04T14:04:26.570Z Skipped 0 0 0 0
93842024-02-04T14:04:26.570Z Error 0 0 0 0
93852024-02-04T14:04:26.570ZLast Flush: 1016 0 1016
93862024-02-04T14:04:26.570ZDownstairs last five completed:
93872024-02-04T14:04:26.570ZUpstairs last five completed: 30 29 28 27 26
93882024-02-04T14:04:26.570Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:25.479781369Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1473}
93892024-02-04T14:04:26.570ZWrote out file "/tmp/test_repair_verify.data"
93902024-02-04T14:04:26.571Z
93912024-02-04T14:04:26.571ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93922024-02-04T14:04:26.571ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93932024-02-04T14:04:26.571Z 0 000-019 86 84 86 137 132 137 T F T <---
93942024-02-04T14:04:26.571Z 1 020-039 84 84 84 133 133 133 F F F
93952024-02-04T14:04:26.571Z 2 040-059 82 82 82 130 130 130 F F F
93962024-02-04T14:04:26.571Z 3 060-079 82 82 82 129 129 129 F F F
93972024-02-04T14:04:26.571Z 4 080-099 86 84 86 138 134 138 F F F <---
93982024-02-04T14:04:26.571Z 5 100-119 84 84 84 134 134 134 F F F
93992024-02-04T14:04:26.571Z 6 120-139 84 84 84 134 134 134 F F F
94002024-02-04T14:04:26.571Z 7 140-159 83 83 83 131 131 131 F F F
94012024-02-04T14:04:26.571Z 8 160-179 83 83 83 131 131 131 F F F
94022024-02-04T14:04:26.571Z 9 180-199 86 78 86 136 124 136 F T F <---
94032024-02-04T14:04:26.571Z 10 200-219 80 80 80 127 127 127 F F F
94042024-02-04T14:04:26.571Z 11 220-239 84 84 84 132 132 132 F F F
94052024-02-04T14:04:26.571Z 12 240-259 84 84 84 132 132 132 F F F
94062024-02-04T14:04:26.571Z 13 260-279 84 84 84 132 132 132 T F T <---
94072024-02-04T14:04:26.571Z 14 280-299 80 80 80 127 127 127 F F F
94082024-02-04T14:04:26.571Z 15 300-319 74 74 74 119 119 119 F F F
94092024-02-04T14:04:26.571Z 16 320-339 86 83 86 137 131 137 F F F <---
94102024-02-04T14:04:26.571Z 17 340-359 83 83 83 131 131 131 F F F
94112024-02-04T14:04:26.571Z 18 360-379 84 84 84 132 132 132 F F F
94122024-02-04T14:04:26.571Z 19 380-399 79 79 79 126 126 126 F F F
94132024-02-04T14:04:26.571Z 20 400-419 81 81 81 128 128 128 T F T <---
94142024-02-04T14:04:26.571Z 21 420-439 86 77 86 138 122 138 F F F <---
94152024-02-04T14:04:26.572Z 22 440-459 86 83 86 138 131 138 F F F <---
94162024-02-04T14:04:26.572Z 23 460-479 85 85 85 135 135 135 F F F
94172024-02-04T14:04:26.572Z 24 480-499 79 79 79 126 126 126 T F T <---
94182024-02-04T14:04:26.572Z 25 500-519 77 77 77 122 122 122 F F F
94192024-02-04T14:04:26.572Z 26 520-539 84 84 84 132 132 132 F F F
94202024-02-04T14:04:26.572Z 27 540-559 83 83 83 131 131 131 F F F
94212024-02-04T14:04:26.572Z 28 560-579 84 84 84 132 132 132 F F F
94222024-02-04T14:04:26.572Z 29 580-599 85 85 85 135 135 135 T F T <---
94232024-02-04T14:04:26.572ZMax gen: 86, Max flush: 138
94242024-02-04T14:04:26.572ZError: Difference in extent metadata found!
94252024-02-04T14:04:26.572ZA Difference in extent metadata is expected here
94262024-02-04T14:04:26.572ZOn loop 43
94272024-02-04T14:04:26.572Z
94282024-02-04T14:04:26.572ZVerifying data now
94292024-02-04T14:04:31.393ZNo change after verify
94302024-02-04T14:04:31.393ZLoop: 43 Downstairs dump after verify (and repair):
94312024-02-04T14:04:31.416ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94322024-02-04T14:04:31.416Z 0 000-019 87 87 87 139 139 139 F F F
94332024-02-04T14:04:31.416Z 1 020-039 84 84 84 133 133 133 F F F
94342024-02-04T14:04:31.416Z 2 040-059 82 82 82 130 130 130 F F F
94352024-02-04T14:04:31.416Z 3 060-079 82 82 82 129 129 129 F F F
94362024-02-04T14:04:31.419Z 4 080-099 86 86 86 138 138 138 F F F
94372024-02-04T14:04:31.419Z 5 100-119 84 84 84 134 134 134 F F F
94382024-02-04T14:04:31.419Z 6 120-139 84 84 84 134 134 134 F F F
94392024-02-04T14:04:31.419Z 7 140-159 83 83 83 131 131 131 F F F
94402024-02-04T14:04:31.419Z 8 160-179 83 83 83 131 131 131 F F F
94412024-02-04T14:04:31.419Z 9 180-199 86 86 86 136 136 136 F F F
94422024-02-04T14:04:31.419Z 10 200-219 80 80 80 127 127 127 F F F
94432024-02-04T14:04:31.419Z 11 220-239 84 84 84 132 132 132 F F F
94442024-02-04T14:04:31.419Z 12 240-259 84 84 84 132 132 132 F F F
94452024-02-04T14:04:31.419Z 13 260-279 87 87 87 139 139 139 F F F
94462024-02-04T14:04:31.420Z 14 280-299 80 80 80 127 127 127 F F F
94472024-02-04T14:04:31.420Z 15 300-319 74 74 74 119 119 119 F F F
94482024-02-04T14:04:31.420Z 16 320-339 86 86 86 137 137 137 F F F
94492024-02-04T14:04:31.420Z 17 340-359 83 83 83 131 131 131 F F F
94502024-02-04T14:04:31.420Z 18 360-379 84 84 84 132 132 132 F F F
94512024-02-04T14:04:31.420Z 19 380-399 79 79 79 126 126 126 F F F
94522024-02-04T14:04:31.420Z 20 400-419 87 87 87 139 139 139 F F F
94532024-02-04T14:04:31.420Z 21 420-439 86 86 86 138 138 138 F F F
94542024-02-04T14:04:31.420Z 22 440-459 86 86 86 138 138 138 F F F
94552024-02-04T14:04:31.420Z 23 460-479 85 85 85 135 135 135 F F F
94562024-02-04T14:04:31.420Z 24 480-499 87 87 87 139 139 139 F F F
94572024-02-04T14:04:31.420Z 25 500-519 77 77 77 122 122 122 F F F
94582024-02-04T14:04:31.420Z 26 520-539 84 84 84 132 132 132 F F F
94592024-02-04T14:04:31.420Z 27 540-559 83 83 83 131 131 131 F F F
94602024-02-04T14:04:31.420Z 28 560-579 84 84 84 132 132 132 F F F
94612024-02-04T14:04:31.420Z 29 580-599 87 87 87 139 139 139 F F F
94622024-02-04T14:04:31.420ZMax gen: 87, Max flush: 139
94632024-02-04T14:04:31.423Z
94642024-02-04T14:04:31.423ZBegin loop 44
94652024-02-04T14:04:31.423ZDownstairs to restart: 0
94662024-02-04T14:04:31.450Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.482965654Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481}
94672024-02-04T14:04:31.450Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.483416286Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481}
94682024-02-04T14:04:31.453Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.483481824Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481}
94692024-02-04T14:04:31.453Z{"msg":"Crucible 68069d40-b942-4e38-99dc-5bbca9f2ce90 has session id: a508f01f-0131-4fde-a28b-ad711dfcc9ad","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.483520101Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94702024-02-04T14:04:31.453Z{"msg":"Upstairs opts: Upstairs UUID: 68069d40-b942-4e38-99dc-5bbca9f2ce90, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.483628814Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94712024-02-04T14:04:31.453Z{"msg":"Crucible stats registered with UUID: 68069d40-b942-4e38-99dc-5bbca9f2ce90","v":0,"name":"crucible","level":30Crucible runtime is spawned
94722024-02-04T14:04:31.453Z,"time":"2024-02-04T14:04:30.483684092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94732024-02-04T14:04:31.453Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.483820899Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481}
94742024-02-04T14:04:31.454Z{"msg":"68069d40-b942-4e38-99dc-5bbca9f2ce90 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.483983053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94752024-02-04T14:04:31.454Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.48407698Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94762024-02-04T14:04:31.454Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484113019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94772024-02-04T14:04:31.454Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484157813Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94782024-02-04T14:04:31.454Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484182622Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94792024-02-04T14:04:31.454Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484309976Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94802024-02-04T14:04:31.454Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484479677Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94812024-02-04T14:04:31.454Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484550748Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94822024-02-04T14:04:31.454Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484618757Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94832024-02-04T14:04:31.454Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484664156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94842024-02-04T14:04:31.454Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484727772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94852024-02-04T14:04:31.454Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484772396Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94862024-02-04T14:04:31.454Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.484866973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94872024-02-04T14:04:31.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.485878208Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94882024-02-04T14:04:31.454Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.48599769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94892024-02-04T14:04:31.454Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.48611724Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94902024-02-04T14:04:31.454Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.486161632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94912024-02-04T14:04:31.458Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.486782683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94922024-02-04T14:04:31.459Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.486851646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94932024-02-04T14:04:31.459Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.487026614Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94942024-02-04T14:04:31.459Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.487206234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94952024-02-04T14:04:31.459Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.488057436Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94962024-02-04T14:04:31.459Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.488120415Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94972024-02-04T14:04:31.459Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.488145249Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94982024-02-04T14:04:31.459Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.488204823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
94992024-02-04T14:04:31.459Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.488345728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95002024-02-04T14:04:31.459Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.488429632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95012024-02-04T14:04:31.459Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.488504693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95022024-02-04T14:04:31.459Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489006371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95032024-02-04T14:04:31.459Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489073276Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95042024-02-04T14:04:31.459Z{"msg":"[0]R flush_numbers[0..12]: [139, 133, 130, 129, 138, 134, 134, 131, 131, 136, 127, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489101505Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95052024-02-04T14:04:31.459Z{"msg":"[0]R generation[0..12]: [87, 84, 82, 82, 86, 84, 84, 83, 83, 86, 80, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.48912546Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95062024-02-04T14:04:31.459Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489169756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95072024-02-04T14:04:31.459Z{"msg":"[1]R flush_numbers[0..12]: [139, 133, 130, 129, 138, 134, 134, 131, 131, 136, 127, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489195475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95082024-02-04T14:04:31.459Z{"msg":"[1]R generation[0..12]: [87, 84, 82, 82, 86, 84, 84, 83, 83, 86, 80, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489219836Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95092024-02-04T14:04:31.459Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489263012Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95102024-02-04T14:04:31.459Z{"msg":"[2]R flush_numbers[0..12]: [139, 133, 130, 129, 138, 134, 134, 131, 131, 136, 127, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489288553Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95112024-02-04T14:04:31.459Z{"msg":"[2]R generation[0..12]: [87, 84, 82, 82, 86, 84, 84, 83, 83, 86, 80, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489311993Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95122024-02-04T14:04:31.459Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
95132024-02-04T14:04:31.459Z,"time":"2024-02-04T14:04:30.489335856Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95142024-02-04T14:04:31.459Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489359959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95152024-02-04T14:04:31.460Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489455371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95162024-02-04T14:04:31.460Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489479795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95172024-02-04T14:04:31.460Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.48952123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95182024-02-04T14:04:31.460Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489545158Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95192024-02-04T14:04:31.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.48956547Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95202024-02-04T14:04:31.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489605893Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95212024-02-04T14:04:31.460Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489629147Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95222024-02-04T14:04:31.460Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489651383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95232024-02-04T14:04:31.460Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489690856Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95242024-02-04T14:04:31.460Z{"msg":"68069d40-b942-4e38-99dc-5bbca9f2ce90 is now active with session: a508f01f-0131-4fde-a28b-ad711dfcc9ad","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489713905Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95252024-02-04T14:04:31.460Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489733926Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481,"session_id":"a508f01f-0131-4fde-a28b-ad711dfcc9ad"}
95262024-02-04T14:04:31.460Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:30.489755709Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481}
95272024-02-04T14:04:31.460ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95282024-02-04T14:04:31.460ZLoading write count information from file "/tmp/test_repair_verify.data"
95292024-02-04T14:04:31.460ZRun Repair workload
95302024-02-04T14:04:31.467Z01/30 Write block 384 len 16384 data: 6 7 6 8
95312024-02-04T14:04:31.467Z02/30 Write block 185 len 36864 data: 11 9 8 7 7 5 5 5 4
95322024-02-04T14:04:31.467Z03/30 Flush
95332024-02-04T14:04:31.470Z04/30 Write block 28 len 32768 data: 6 6 6 7 5 6 5 6
95342024-02-04T14:04:31.470Z05/30 Write block 280 len 4096 data: 5
95352024-02-04T14:04:31.470Z06/30 Read block 362 len 4096
95362024-02-04T14:04:31.470Z07/30 Read block 99 len 8192
95372024-02-04T14:04:31.470Z08/30 Flush
95382024-02-04T14:04:31.473Z09/30 Write block 56 len 8192 data: 5 6
95392024-02-04T14:04:31.473Z10/30 Read block 149 len 16384
95402024-02-04T14:04:31.475Z11/30 Write block 41 len 24576 data: 4 6 6 7 7 7
95412024-02-04T14:04:31.478Z12/30 Write block 112 len 40960 data: 13 11 11 11 10 11 8 8 8 8
95422024-02-04T14:04:31.478Z13/30 Write block 184 len 36864 data: 10 12 10 9 8 8 6 6 6
95432024-02-04T14:04:31.478Z14/30 Read block 238 len 36864
95442024-02-04T14:04:31.481Z15/30 Read block 476 len 28672
95452024-02-04T14:04:31.484Z16/30 Write block 273 len 36864 data: 4 4 4 4 4 6 6 6 6
95462024-02-04T14:04:31.484Z17/30 Read block 168 len 36864
95472024-02-04T14:04:31.486Z18/30 Read block 510 len 36864
95482024-02-04T14:04:31.489Z19/30 Write block 345 len 28672 data: 7 6 7 6 8 6 8
95492024-02-04T14:04:31.489Z20/30 Read block 155 len 8192
95502024-02-04T14:04:31.492Z21/30 Read block 541 len 20480
95512024-02-04T14:04:31.493Z22/30 Flush
95522024-02-04T14:04:31.495Z23/30 Write block 179 len 4096 data: 10
95532024-02-04T14:04:31.498Z24/30 Read block 522 len 12288
95542024-02-04T14:04:31.498Z25/30 Flush
95552024-02-04T14:04:31.501Z26/30 Write block 212 len 4096 data: 5
95562024-02-04T14:04:31.501Z27/30 Write block 92 len 24576 data: 4 2 2 5 5 6
95572024-02-04T14:04:31.501Z28/30 Write block 446 len 12288 data: 8 8 7
95582024-02-04T14:04:31.504Z29/30 Write block 490 len 16384 data: 9 10 12 12
95592024-02-04T14:04:31.504Z30/30 Read block 332 len 8192
95602024-02-04T14:04:31.507Z----------------------------------------------------------------
95612024-02-04T14:04:31.507Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95622024-02-04T14:04:31.507ZStates: Active Active Active
95632024-02-04T14:04:31.507ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95642024-02-04T14:04:31.507Z 1 Acked 1000 Write 4 Done Done Done false
95652024-02-04T14:04:31.507Z 2 Acked 1001 Write 9 Sent Done Done false
95662024-02-04T14:04:31.508Z 3 Acked 1002 Flush 0 Sent Done Done false
95672024-02-04T14:04:31.508Z 4 Acked 1003 Write 8 Sent Done Done false
95682024-02-04T14:04:31.508Z 5 Acked 1004 Write 1 Sent Done Done false
95692024-02-04T14:04:31.508Z 6 Acked 1005 Read 1 Sent Done Done false
95702024-02-04T14:04:31.508Z 7 Acked 1006 Read 2 Sent Done Done false
95712024-02-04T14:04:31.508Z 8 Acked 1007 Flush 0 Sent Done Done false
95722024-02-04T14:04:31.508Z 9 Acked 1008 Write 2 Sent Done Done false
95732024-02-04T14:04:31.508Z 10 Acked 1009 Read 4 Sent Done Done false
95742024-02-04T14:04:31.508Z 11 Acked 1010 Write 6 Sent Done Done false
95752024-02-04T14:04:31.508Z 12 Acked 1011 Write 10 Sent Done Done false
95762024-02-04T14:04:31.508Z 13 Acked 1012 Write 9 Sent Done Done false
95772024-02-04T14:04:31.508Z 14 Acked 1013 Read 9 Sent Done Done false
95782024-02-04T14:04:31.508Z 15 Acked 1014 Read 7 Sent Done Done false
95792024-02-04T14:04:31.508Z 16 Acked 1015 Write 9 Sent Done Done false
95802024-02-04T14:04:31.508Z 17 Acked 1016 Read 9 Sent Done Done false
95812024-02-04T14:04:31.508Z 18 Acked 1017 Read 9 Sent Done Done false
95822024-02-04T14:04:31.508Z 19 Acked 1018 Write 7 Sent Done Done false
95832024-02-04T14:04:31.508Z 20 Acked 1019 Read 2 Sent Done Done false
95842024-02-04T14:04:31.508Z 21 Acked 1020 Read 5 Sent Done Done false
95852024-02-04T14:04:31.508Z 22 Acked 1021 Flush 0 Sent Done Done false
95862024-02-04T14:04:31.508Z 23 Acked 1022 Write 1 Sent Done Done false
95872024-02-04T14:04:31.508Z 24 Acked 1023 Read 3 Sent Done Done false
95882024-02-04T14:04:31.509Z 25 Acked 1024 Flush 0 Sent Done Done false
95892024-02-04T14:04:31.509Z 26 Acked 1025 Write 1 Sent Done Done false
95902024-02-04T14:04:31.509Z 27 Acked 1026 Write 6 Sent Done Done false
95912024-02-04T14:04:31.509Z 28 Acked 1027 Write 3 Sent Done Done false
95922024-02-04T14:04:31.509Z 29 Acked 1028 Write 4 Sent Done Done false
95932024-02-04T14:04:31.509Z 30 Acked 1029 Read 2 Sent Done Done false
95942024-02-04T14:04:31.509Z STATES DS:0 DS:1 DS:2 TOTAL
95952024-02-04T14:04:31.509Z New 0 0 0 0
95962024-02-04T14:04:31.509Z Sent 29 0 0 29
95972024-02-04T14:04:31.509Z Done 1 30 30 61
95982024-02-04T14:04:31.509Z Skipped 0 0 0 0
95992024-02-04T14:04:31.509Z Error 0 0 0 0
96002024-02-04T14:04:31.509ZLast Flush: 0 1024 1024
96012024-02-04T14:04:31.509ZDownstairs last five completed:
96022024-02-04T14:04:31.509ZUpstairs last five completed: 30 29 28 27 26
96032024-02-04T14:04:31.509Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:30.538279168Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1481}
96042024-02-04T14:04:31.509ZWrote out file "/tmp/test_repair_verify.data"
96052024-02-04T14:04:31.512Z
96062024-02-04T14:04:31.519ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96072024-02-04T14:04:31.540ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96082024-02-04T14:04:31.540Z 0 000-019 87 87 87 139 139 139 F F F
96092024-02-04T14:04:31.540Z 1 020-039 84 88 88 133 141 141 F F F <---
96102024-02-04T14:04:31.540Z 2 040-059 82 88 88 130 142 142 F F F <---
96112024-02-04T14:04:31.541Z 3 060-079 82 82 82 129 129 129 F F F
96122024-02-04T14:04:31.544Z 4 080-099 86 86 86 138 138 138 F T T <---
96132024-02-04T14:04:31.544Z 5 100-119 84 88 88 134 142 142 F F F <---
96142024-02-04T14:04:31.544Z 6 120-139 84 88 88 134 142 142 F F F <---
96152024-02-04T14:04:31.544Z 7 140-159 83 83 83 131 131 131 F F F
96162024-02-04T14:04:31.544Z 8 160-179 83 88 88 131 143 143 F F F <---
96172024-02-04T14:04:31.544Z 9 180-199 86 88 88 136 142 142 F F F <---
96182024-02-04T14:04:31.544Z 10 200-219 80 80 80 127 127 127 F T T <---
96192024-02-04T14:04:31.544Z 11 220-239 84 84 84 132 132 132 F F F
96202024-02-04T14:04:31.544Z 12 240-259 84 84 84 132 132 132 F F F
96212024-02-04T14:04:31.544Z 13 260-279 87 88 88 139 142 142 F F F <---
96222024-02-04T14:04:31.544Z 14 280-299 80 88 88 127 142 142 F F F <---
96232024-02-04T14:04:31.544Z 15 300-319 74 74 74 119 119 119 F F F
96242024-02-04T14:04:31.544Z 16 320-339 86 86 86 137 137 137 F F F
96252024-02-04T14:04:31.545Z 17 340-359 83 88 88 131 142 142 F F F <---
96262024-02-04T14:04:31.545Z 18 360-379 84 84 84 132 132 132 F F F
96272024-02-04T14:04:31.545Z 19 380-399 79 88 88 126 140 140 T F F <---
96282024-02-04T14:04:31.545Z 20 400-419 87 87 87 139 139 139 F F F
96292024-02-04T14:04:31.545Z 21 420-439 86 86 86 138 138 138 F F F
96302024-02-04T14:04:31.545Z 22 440-459 86 86 86 138 138 138 F T T <---
96312024-02-04T14:04:31.545Z 23 460-479 85 85 85 135 135 135 F F F
96322024-02-04T14:04:31.545Z 24 480-499 87 87 87 139 139 139 F T T <---
96332024-02-04T14:04:31.545Z 25 500-519 77 77 77 122 122 122 F F F
96342024-02-04T14:04:31.545Z 26 520-539 84 84 84 132 132 132 F F F
96352024-02-04T14:04:31.545Z 27 540-559 83 83 83 131 131 131 F F F
96362024-02-04T14:04:31.545Z 28 560-579 84 84 84 132 132 132 F F F
96372024-02-04T14:04:31.545Z 29 580-599 87 87 87 139 139 139 F F F
96382024-02-04T14:04:31.545ZMax gen: 88, Max flush: 143
96392024-02-04T14:04:31.545ZError: Difference in extent metadata found!
96402024-02-04T14:04:31.548ZA Difference in extent metadata is expected here
96412024-02-04T14:04:31.548ZOn loop 44
96422024-02-04T14:04:31.548Z
96432024-02-04T14:04:31.548ZVerifying data now
96442024-02-04T14:04:36.755ZNo change after verify
96452024-02-04T14:04:36.755ZLoop: 44 Downstairs dump after verify (and repair):
96462024-02-04T14:04:36.777ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96472024-02-04T14:04:36.778Z 0 000-019 87 87 87 139 139 139 F F F
96482024-02-04T14:04:36.778Z 1 020-039 88 88 88 141 141 141 F F F
96492024-02-04T14:04:36.778Z 2 040-059 88 88 88 142 142 142 F F F
96502024-02-04T14:04:36.778Z 3 060-079 82 82 82 129 129 129 F F F
96512024-02-04T14:04:36.778Z 4 080-099 89 89 89 144 144 144 F F F
96522024-02-04T14:04:36.778Z 5 100-119 88 88 88 142 142 142 F F F
96532024-02-04T14:04:36.781Z 6 120-139 88 88 88 142 142 142 F F F
96542024-02-04T14:04:36.781Z 7 140-159 83 83 83 131 131 131 F F F
96552024-02-04T14:04:36.782Z 8 160-179 88 88 88 143 143 143 F F F
96562024-02-04T14:04:36.782Z 9 180-199 88 88 88 142 142 142 F F F
96572024-02-04T14:04:36.782Z 10 200-219 89 89 89 144 144 144 F F F
96582024-02-04T14:04:36.782Z 11 220-239 84 84 84 132 132 132 F F F
96592024-02-04T14:04:36.782Z 12 240-259 84 84 84 132 132 132 F F F
96602024-02-04T14:04:36.782Z 13 260-279 88 88 88 142 142 142 F F F
96612024-02-04T14:04:36.782Z 14 280-299 88 88 88 142 142 142 F F F
96622024-02-04T14:04:36.782Z 15 300-319 74 74 74 119 119 119 F F F
96632024-02-04T14:04:36.782Z 16 320-339 86 86 86 137 137 137 F F F
96642024-02-04T14:04:36.782Z 17 340-359 88 88 88 142 142 142 F F F
96652024-02-04T14:04:36.782Z 18 360-379 84 84 84 132 132 132 F F F
96662024-02-04T14:04:36.782Z 19 380-399 88 88 88 140 140 140 F F F
96672024-02-04T14:04:36.782Z 20 400-419 87 87 87 139 139 139 F F F
96682024-02-04T14:04:36.783Z 21 420-439 86 86 86 138 138 138 F F F
96692024-02-04T14:04:36.783Z 22 440-459 89 89 89 144 144 144 F F F
96702024-02-04T14:04:36.783Z 23 460-479 85 85 85 135 135 135 F F F
96712024-02-04T14:04:36.783Z 24 480-499 89 89 89 144 144 144 F F F
96722024-02-04T14:04:36.783Z 25 500-519 77 77 77 122 122 122 F F F
96732024-02-04T14:04:36.783Z 26 520-539 84 84 84 132 132 132 F F F
96742024-02-04T14:04:36.783Z 27 540-559 83 83 83 131 131 131 F F F
96752024-02-04T14:04:36.783Z 28 560-579 84 84 84 132 132 132 F F F
96762024-02-04T14:04:36.783Z 29 580-599 87 87 87 139 139 139 F F F
96772024-02-04T14:04:36.783ZMax gen: 89, Max flush: 144
96782024-02-04T14:04:36.798Z
96792024-02-04T14:04:36.799ZBegin loop 45
96802024-02-04T14:04:36.799ZDownstairs to restart: 2
96812024-02-04T14:04:36.813Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.845706292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489}
96822024-02-04T14:04:36.816Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.846275352Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489}
96832024-02-04T14:04:36.816Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.846404528Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489}
96842024-02-04T14:04:36.816Z{"msg":"Crucible 705d5063-0dfd-4d86-a802-06c7bd3a44fa has session id: fe2d439d-02f1-450a-8ac2-08d2a9b6b009","v":0,"name":"crucible","level":30Crucible runtime is spawned
96852024-02-04T14:04:36.816Z,"time":"2024-02-04T14:04:35.846443326Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96862024-02-04T14:04:36.816Z{"msg":"Upstairs opts: Upstairs UUID: 705d5063-0dfd-4d86-a802-06c7bd3a44fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.846539825Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96872024-02-04T14:04:36.816Z{"msg":"Crucible stats registered with UUID: 705d5063-0dfd-4d86-a802-06c7bd3a44fa","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.84660904Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96882024-02-04T14:04:36.817Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.846641826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489}
96892024-02-04T14:04:36.817Z{"msg":"705d5063-0dfd-4d86-a802-06c7bd3a44fa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.846915532Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96902024-02-04T14:04:36.817Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.846966114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96912024-02-04T14:04:36.817Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.846991561Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96922024-02-04T14:04:36.817Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847041554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96932024-02-04T14:04:36.817Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847067634Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96942024-02-04T14:04:36.817Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847138272Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96952024-02-04T14:04:36.817Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847164969Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96962024-02-04T14:04:36.817Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847192474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"io task","client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96972024-02-04T14:04:36.817Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847262089Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"io task","client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96982024-02-04T14:04:36.817Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847308772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"io task","client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
96992024-02-04T14:04:36.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847385021Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"io task","client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97002024-02-04T14:04:36.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.847589478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"io task","client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97012024-02-04T14:04:36.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.84770244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"io task","client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97022024-02-04T14:04:36.818Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.848716824Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97032024-02-04T14:04:36.818Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.848825115Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97042024-02-04T14:04:36.818Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.848957416Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97052024-02-04T14:04:36.818Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.849098396Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97062024-02-04T14:04:36.822Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.849748143Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97072024-02-04T14:04:36.822Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.850022087Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97082024-02-04T14:04:36.823Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.850207765Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97092024-02-04T14:04:36.823Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.850289482Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97102024-02-04T14:04:36.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.850693928Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97112024-02-04T14:04:36.823Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.850750217Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97122024-02-04T14:04:36.823Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.850794723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97132024-02-04T14:04:36.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.85085539Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97142024-02-04T14:04:36.823Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.850904848Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97152024-02-04T14:04:36.823Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.850961364Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97162024-02-04T14:04:36.823Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.851164869Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97172024-02-04T14:04:36.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.851826892Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97182024-02-04T14:04:36.823Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.851904122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97192024-02-04T14:04:36.823Z{"msg":"[0]R flush_numbers[0..12]: [139, 141, 142, 129, 144, 142, 142, 131, 143, 142, 144, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.851938783Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97202024-02-04T14:04:36.824Z{"msg":"[0]R generation[0..12]: [87, 88, 88, 82, 89, 88, 88, 83, 88, 88, 89, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.851996754Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97212024-02-04T14:04:36.824Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852031784Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97222024-02-04T14:04:36.824Z{"msg":"[1]R flush_numbers[0..12]: [139, 141, 142, 129, 144, 142, 142, 131, 143, 142, 144, 132]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
97232024-02-04T14:04:36.824Z,"time":"2024-02-04T14:04:35.852089381Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97242024-02-04T14:04:36.824Z{"msg":"[1]R generation[0..12]: [87, 88, 88, 82, 89, 88, 88, 83, 88, 88, 89, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852125231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97252024-02-04T14:04:36.824Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852182949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97262024-02-04T14:04:36.824Z{"msg":"[2]R flush_numbers[0..12]: [139, 141, 142, 129, 144, 142, 142, 131, 143, 142, 144, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852216511Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97272024-02-04T14:04:36.824Z{"msg":"[2]R generation[0..12]: [87, 88, 88, 82, 89, 88, 88, 83, 88, 88, 89, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852293892Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97282024-02-04T14:04:36.824Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852338004Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97292024-02-04T14:04:36.824Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852431475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97302024-02-04T14:04:36.824Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852484709Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97312024-02-04T14:04:36.824Z{"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852508843Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97322024-02-04T14:04:36.824Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852531129Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97332024-02-04T14:04:36.825Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852575271Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97342024-02-04T14:04:36.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852611028Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97352024-02-04T14:04:36.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.85268339Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97362024-02-04T14:04:36.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852757948Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97372024-02-04T14:04:36.825ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97382024-02-04T14:04:36.825Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852835227Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97392024-02-04T14:04:36.825Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852897844Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97402024-02-04T14:04:36.825Z{"msg":"705d5063-0dfd-4d86-a802-06c7bd3a44fa is now active with session: fe2d439d-02f1-450a-8ac2-08d2a9b6b009","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852927106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97412024-02-04T14:04:36.825Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852967184Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489,"session_id":"fe2d439d-02f1-450a-8ac2-08d2a9b6b009"}
97422024-02-04T14:04:36.825Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:35.852989448Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489}
97432024-02-04T14:04:36.825ZLoading write count information from file "/tmp/test_repair_verify.data"
97442024-02-04T14:04:36.825ZRun Repair workload
97452024-02-04T14:04:36.826Z01/30 Write block 97 len 4096 data: 7
97462024-02-04T14:04:36.829Z02/30 Read block 415 len 4096
97472024-02-04T14:04:36.829Z03/30 Write block 366 len 36864 data: 7 9 8 8 9 8 5 5 8
97482024-02-04T14:04:36.829Z04/30 Write block 143 len 36864 data: 3 6 7 7 6 6 7 7 7
97492024-02-04T14:04:36.829Z05/30 Read block 321 len 40960
97502024-02-04T14:04:36.830Z06/30 Read block 231 len 4096
97512024-02-04T14:04:36.833Z07/30 Read block 330 len 36864
97522024-02-04T14:04:36.833Z08/30 Read block 35 len 20480
97532024-02-04T14:04:36.841Z09/30 Read block 264 len 40960
97542024-02-04T14:04:36.841Z10/30 Read block 57 len 8192
97552024-02-04T14:04:36.841Z11/30 Read block 558 len 24576
97562024-02-04T14:04:36.844Z12/30 Read block 491 len 12288
97572024-02-04T14:04:36.844Z13/30 Read block 355 len 8192
97582024-02-04T14:04:36.844Z14/30 Read block 138 len 24576
97592024-02-04T14:04:36.844Z15/30 Read block 96 len 20480
97602024-02-04T14:04:36.844Z16/30 Write block 220 len 12288 data: 7 8 6
97612024-02-04T14:04:36.844Z17/30 Write block 393 len 12288 data: 12 11 10
97622024-02-04T14:04:36.844Z18/30 Flush
97632024-02-04T14:04:36.849Z19/30 Write block 593 len 20480 data: 8 9 9 5 3
97642024-02-04T14:04:36.849Z20/30 Write block 496 len 4096 data: 11
97652024-02-04T14:04:36.852Z21/30 Flush
97662024-02-04T14:04:36.852Z22/30 Write block 402 len 8192 data: 9 6
97672024-02-04T14:04:36.855Z23/30 Flush
97682024-02-04T14:04:36.855Z24/30 Write block 35 len 16384 data: 7 8 7 5
97692024-02-04T14:04:36.858Z25/30 Read block 378 len 28672
97702024-02-04T14:04:36.859Z26/30 Read block 473 len 16384
97712024-02-04T14:04:36.859Z27/30 Read block 478 len 24576
97722024-02-04T14:04:36.862Z28/30 Read block 510 len 40960
97732024-02-04T14:04:36.865Z29/30 Read block 233 len 36864
97742024-02-04T14:04:36.865Z30/30 Read block 27 len 40960
97752024-02-04T14:04:36.869Z----------------------------------------------------------------
97762024-02-04T14:04:36.869Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
97772024-02-04T14:04:36.869ZStates: Active Active Active
97782024-02-04T14:04:36.869ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97792024-02-04T14:04:36.869Z 1 Acked 1000 Write 1 Done Done Sent false
97802024-02-04T14:04:36.869Z 2 Acked 1001 Read 1 Done Done Sent false
97812024-02-04T14:04:36.869Z 3 Acked 1002 Write 9 Done Done Sent false
97822024-02-04T14:04:36.869Z 4 Acked 1003 Write 9 Done Done Sent false
97832024-02-04T14:04:36.869Z 5 Acked 1004 Read 10 Done Done Sent false
97842024-02-04T14:04:36.869Z 6 Acked 1005 Read 1 Done Done Sent false
97852024-02-04T14:04:36.869Z 7 Acked 1006 Read 9 Done Done Sent false
97862024-02-04T14:04:36.869Z 8 Acked 1007 Read 5 Done Done Sent false
97872024-02-04T14:04:36.870Z 9 Acked 1008 Read 10 Done Done Sent false
97882024-02-04T14:04:36.870Z 10 Acked 1009 Read 2 Done Done Sent false
97892024-02-04T14:04:36.870Z 11 Acked 1010 Read 6 Done Done Sent false
97902024-02-04T14:04:36.870Z 12 Acked 1011 Read 3 Done Done Sent false
97912024-02-04T14:04:36.870Z 13 Acked 1012 Read 2 Done Done Sent false
97922024-02-04T14:04:36.870Z 14 Acked 1013 Read 6 Done Done Sent false
97932024-02-04T14:04:36.870Z 15 Acked 1014 Read 5 Done Done Sent false
97942024-02-04T14:04:36.870Z 16 Acked 1015 Write 3 Done Done Sent false
97952024-02-04T14:04:36.870Z 17 Acked 1016 Write 3 Done Done Sent false
97962024-02-04T14:04:36.870Z 18 Acked 1017 Flush 0 Done Done Sent false
97972024-02-04T14:04:36.870Z 19 Acked 1018 Write 5 Done Done Sent false
97982024-02-04T14:04:36.870Z 20 Acked 1019 Write 1 Done Done Sent false
97992024-02-04T14:04:36.870Z 21 Acked 1020 Flush 0 Done Done Sent false
98002024-02-04T14:04:36.871Z 22 Acked 1021 Write 2 Done Done Sent false
98012024-02-04T14:04:36.871Z 23 Acked 1022 Flush 0 Done Done Sent false
98022024-02-04T14:04:36.871Z 24 Acked 1023 Write 4 Done Done Sent false
98032024-02-04T14:04:36.871Z 25 Acked 1024 Read 7 Done Done Sent false
98042024-02-04T14:04:36.871Z 26 Acked 1025 Read 4 Done Done Sent false
98052024-02-04T14:04:36.871Z 27 Acked 1026 Read 6 Done Done Sent false
98062024-02-04T14:04:36.871Z 28 Acked 1027 Read 10 Done Done Sent false
98072024-02-04T14:04:36.871Z 29 Acked 1028 Read 9 Done Done Sent false
98082024-02-04T14:04:36.871Z 30 Acked 1029 Read 10 Done Sent Sent false
98092024-02-04T14:04:36.871Z STATES DS:0 DS:1 DS:2 TOTAL
98102024-02-04T14:04:36.871Z New 0 0 0 0
98112024-02-04T14:04:36.871Z Sent 0 1 30 31
98122024-02-04T14:04:36.871Z Done 30 29 0 59
98132024-02-04T14:04:36.872Z Skipped 0 0 0 0
98142024-02-04T14:04:36.872Z Error 0 0 0 0
98152024-02-04T14:04:36.872ZLast Flush: 1022 1022 0
98162024-02-04T14:04:36.872ZDownstairs last five completed:
98172024-02-04T14:04:36.872ZUpstairs last five completed: 30 29 28 27 26
98182024-02-04T14:04:36.872Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:35.899162135Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1489}
98192024-02-04T14:04:36.872ZWrote out file "/tmp/test_repair_verify.data"
98202024-02-04T14:04:36.875Z
98212024-02-04T14:04:36.879ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98222024-02-04T14:04:36.897ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98232024-02-04T14:04:36.897Z 0 000-019 87 87 87 139 139 139 F F F
98242024-02-04T14:04:36.898Z 1 020-039 88 88 88 141 141 141 T T F <---
98252024-02-04T14:04:36.898Z 2 040-059 88 88 88 142 142 142 F F F
98262024-02-04T14:04:36.898Z 3 060-079 82 82 82 129 129 129 F F F
98272024-02-04T14:04:36.898Z 4 080-099 90 90 89 145 145 144 F F F <---
98282024-02-04T14:04:36.898Z 5 100-119 88 88 88 142 142 142 F F F
98292024-02-04T14:04:36.898Z 6 120-139 88 88 88 142 142 142 F F F
98302024-02-04T14:04:36.898Z 7 140-159 90 90 83 145 145 131 F F F <---
98312024-02-04T14:04:36.898Z 8 160-179 88 88 88 143 143 143 F F F
98322024-02-04T14:04:36.901Z 9 180-199 88 88 88 142 142 142 F F F
98332024-02-04T14:04:36.901Z 10 200-219 89 89 89 144 144 144 F F F
98342024-02-04T14:04:36.901Z 11 220-239 90 90 84 145 145 132 F F F <---
98352024-02-04T14:04:36.901Z 12 240-259 84 84 84 132 132 132 F F F
98362024-02-04T14:04:36.901Z 13 260-279 88 88 88 142 142 142 F F F
98372024-02-04T14:04:36.901Z 14 280-299 88 88 88 142 142 142 F F F
98382024-02-04T14:04:36.901Z 15 300-319 74 74 74 119 119 119 F F F
98392024-02-04T14:04:36.901Z 16 320-339 86 86 86 137 137 137 F F F
98402024-02-04T14:04:36.901Z 17 340-359 88 88 88 142 142 142 F F F
98412024-02-04T14:04:36.901Z 18 360-379 90 90 84 145 145 132 F F F <---
98422024-02-04T14:04:36.901Z 19 380-399 90 90 88 145 145 140 F F F <---
98432024-02-04T14:04:36.902Z 20 400-419 90 90 87 147 147 139 F F F <---
98442024-02-04T14:04:36.902Z 21 420-439 86 86 86 138 138 138 F F F
98452024-02-04T14:04:36.902Z 22 440-459 89 89 89 144 144 144 F F F
98462024-02-04T14:04:36.902Z 23 460-479 85 85 85 135 135 135 F F F
98472024-02-04T14:04:36.902Z 24 480-499 90 90 89 146 146 144 F F F <---
98482024-02-04T14:04:36.902Z 25 500-519 77 77 77 122 122 122 F F F
98492024-02-04T14:04:36.902Z 26 520-539 84 84 84 132 132 132 F F F
98502024-02-04T14:04:36.902Z 27 540-559 83 83 83 131 131 131 F F F
98512024-02-04T14:04:36.902Z 28 560-579 84 84 84 132 132 132 F F F
98522024-02-04T14:04:36.902Z 29 580-599 90 90 87 146 146 139 F F F <---
98532024-02-04T14:04:36.902ZMax gen: 90, Max flush: 147
98542024-02-04T14:04:36.902ZError: Difference in extent metadata found!
98552024-02-04T14:04:36.906ZA Difference in extent metadata is expected here
98562024-02-04T14:04:36.906ZOn loop 45
98572024-02-04T14:04:36.906Z
98582024-02-04T14:04:36.906ZVerifying data now
98592024-02-04T14:04:41.727ZNo change after verify
98602024-02-04T14:04:41.727ZLoop: 45 Downstairs dump after verify (and repair):
98612024-02-04T14:04:41.750ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98622024-02-04T14:04:41.750Z 0 000-019 87 87 87 139 139 139 F F F
98632024-02-04T14:04:41.750Z 1 020-039 91 91 91 148 148 148 F F F
98642024-02-04T14:04:41.750Z 2 040-059 88 88 88 142 142 142 F F F
98652024-02-04T14:04:41.750Z 3 060-079 82 82 82 129 129 129 F F F
98662024-02-04T14:04:41.750Z 4 080-099 90 90 90 145 145 145 F F F
98672024-02-04T14:04:41.750Z 5 100-119 88 88 88 142 142 142 F F F
98682024-02-04T14:04:41.750Z 6 120-139 88 88 88 142 142 142 F F F
98692024-02-04T14:04:41.751Z 7 140-159 90 90 90 145 145 145 F F F
98702024-02-04T14:04:41.751Z 8 160-179 88 88 88 143 143 143 F F F
98712024-02-04T14:04:41.754Z 9 180-199 88 88 88 142 142 142 F F F
98722024-02-04T14:04:41.754Z 10 200-219 89 89 89 144 144 144 F F F
98732024-02-04T14:04:41.754Z 11 220-239 90 90 90 145 145 145 F F F
98742024-02-04T14:04:41.754Z 12 240-259 84 84 84 132 132 132 F F F
98752024-02-04T14:04:41.754Z 13 260-279 88 88 88 142 142 142 F F F
98762024-02-04T14:04:41.754Z 14 280-299 88 88 88 142 142 142 F F F
98772024-02-04T14:04:41.754Z 15 300-319 74 74 74 119 119 119 F F F
98782024-02-04T14:04:41.754Z 16 320-339 86 86 86 137 137 137 F F F
98792024-02-04T14:04:41.754Z 17 340-359 88 88 88 142 142 142 F F F
98802024-02-04T14:04:41.754Z 18 360-379 90 90 90 145 145 145 F F F
98812024-02-04T14:04:41.754Z 19 380-399 90 90 90 145 145 145 F F F
98822024-02-04T14:04:41.754Z 20 400-419 90 90 90 147 147 147 F F F
98832024-02-04T14:04:41.754Z 21 420-439 86 86 86 138 138 138 F F F
98842024-02-04T14:04:41.754Z 22 440-459 89 89 89 144 144 144 F F F
98852024-02-04T14:04:41.754Z 23 460-479 85 85 85 135 135 135 F F F
98862024-02-04T14:04:41.754Z 24 480-499 90 90 90 146 146 146 F F F
98872024-02-04T14:04:41.754Z 25 500-519 77 77 77 122 122 122 F F F
98882024-02-04T14:04:41.754Z 26 520-539 84 84 84 132 132 132 F F F
98892024-02-04T14:04:41.755Z 27 540-559 83 83 83 131 131 131 F F F
98902024-02-04T14:04:41.755Z 28 560-579 84 84 84 132 132 132 F F F
98912024-02-04T14:04:41.755Z 29 580-599 90 90 90 146 146 146 F F F
98922024-02-04T14:04:41.755ZMax gen: 91, Max flush: 148
98932024-02-04T14:04:41.758Z
98942024-02-04T14:04:41.758ZBegin loop 46
98952024-02-04T14:04:41.758ZDownstairs to restart: 2
98962024-02-04T14:04:41.784Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.816981082Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497}
98972024-02-04T14:04:41.788Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.817495763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497}
98982024-02-04T14:04:41.788Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.817627509Z","hostname":"Crucible runtime is spawned
98992024-02-04T14:04:41.788Zip-10-150-1-226.us-west-2.compute.internal","pid":1497}
99002024-02-04T14:04:41.788Z{"msg":"Crucible 855d7924-9b4a-4944-b33b-e14a4ef6b63c has session id: 0b06a82d-cd82-4753-acdd-b24c9cb9dfc8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.817753668Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99012024-02-04T14:04:41.788Z{"msg":"Upstairs opts: Upstairs UUID: 855d7924-9b4a-4944-b33b-e14a4ef6b63c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.817807769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99022024-02-04T14:04:41.788Z{"msg":"Crucible stats registered with UUID: 855d7924-9b4a-4944-b33b-e14a4ef6b63c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.817856055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99032024-02-04T14:04:41.788Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.817879809Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497}
99042024-02-04T14:04:41.788Z{"msg":"855d7924-9b4a-4944-b33b-e14a4ef6b63c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.817986088Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99052024-02-04T14:04:41.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.81805053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99062024-02-04T14:04:41.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818104984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99072024-02-04T14:04:41.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818147456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99082024-02-04T14:04:41.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818258177Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99092024-02-04T14:04:41.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818399162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99102024-02-04T14:04:41.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818589125Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99112024-02-04T14:04:41.788Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818655891Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"io task","client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99122024-02-04T14:04:41.788Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818686497Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"io task","client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99132024-02-04T14:04:41.788Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818800561Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"io task","client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99142024-02-04T14:04:41.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818849524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"io task","client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99152024-02-04T14:04:41.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.81891792Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"io task","client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99162024-02-04T14:04:41.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.818950983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"io task","client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99172024-02-04T14:04:41.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.819951794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99182024-02-04T14:04:41.788Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.820089638Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99192024-02-04T14:04:41.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.820133302Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99202024-02-04T14:04:41.789Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.820209693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99212024-02-04T14:04:41.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.820241348Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99222024-02-04T14:04:41.789Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.820340961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99232024-02-04T14:04:41.793Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.821193598Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99242024-02-04T14:04:41.793Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.821303347Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99252024-02-04T14:04:41.793Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.821479447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99262024-02-04T14:04:41.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822159692Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99272024-02-04T14:04:41.793Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822238057Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99282024-02-04T14:04:41.793Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822264695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99292024-02-04T14:04:41.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822290672Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99302024-02-04T14:04:41.793Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822356689Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99312024-02-04T14:04:41.793Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822400669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99322024-02-04T14:04:41.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822446969Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99332024-02-04T14:04:41.793Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822471803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99342024-02-04T14:04:41.793Z{"msg":"[0]R flush_numbers[0..12]: [139, 148, 142, 129, 145, 142, 142, 145, 143, 142, 144, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.82249614Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99352024-02-04T14:04:41.793Z{"msg":"[0]R generation[0..12]: [87, 91, 88, 82, 90, 88, 88, 90, 88, 88, 89, 90]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822519817Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99362024-02-04T14:04:41.793Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822561651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99372024-02-04T14:04:41.793Z{"msg":"[1]R flush_numbers[0..12]: [139, 148, 142, 129, 145, 142, 142, 145, 143, 142, 144, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822586771Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99382024-02-04T14:04:41.793Z{"msg":"[1]R generation[0..12]: [87, 91, 88, 82, 90, 88, 88, 90, 88, 88, 89, 90]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822609692Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99392024-02-04T14:04:41.793Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822633289Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99402024-02-04T14:04:41.793Z{"msg":"[2]R flush_numbers[0..12]: [139, 148, 142, 129, 145, 142, 142, 145, 143, 142, 144, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822657076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99412024-02-04T14:04:41.794Z{"msg":"[2]R generation[0..12]: [87, 91, 88, 82, 90, 88, 88, 90, 88, 88, 89, 90]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822698712Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99422024-02-04T14:04:41.794Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822723381Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99432024-02-04T14:04:41.794Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822745825Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497Wait for a query_work_queue command to finish before sending IO
99442024-02-04T14:04:41.794Z,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99452024-02-04T14:04:41.794Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822829071Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99462024-02-04T14:04:41.794Z{"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822875423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99472024-02-04T14:04:41.794Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822899068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99482024-02-04T14:04:41.794Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822921744Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99492024-02-04T14:04:41.794Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822959985Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99502024-02-04T14:04:41.794Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.822983032Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99512024-02-04T14:04:41.794Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.823022333Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99522024-02-04T14:04:41.794Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.823061507Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99532024-02-04T14:04:41.794Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.823116356Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99542024-02-04T14:04:41.794Z{"msg":"855d7924-9b4a-4944-b33b-e14a4ef6b63c is now active with session: 0b06a82d-cd82-4753-acdd-b24c9cb9dfc8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.823145892Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99552024-02-04T14:04:41.794Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.823197383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497,"session_id":"0b06a82d-cd82-4753-acdd-b24c9cb9dfc8"}
99562024-02-04T14:04:41.794Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:40.823221099Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497}
99572024-02-04T14:04:41.794ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99582024-02-04T14:04:41.794ZLoading write count information from file "/tmp/test_repair_verify.data"
99592024-02-04T14:04:41.794ZRun Repair workload
99602024-02-04T14:04:41.794Z01/30 Write block 403 len 40960 data: 7 6 9 8 8 7 8 8 10 9
99612024-02-04T14:04:41.797Z02/30 Read block 397 len 40960
99622024-02-04T14:04:41.797Z03/30 Read block 446 len 40960
99632024-02-04T14:04:41.800Z04/30 Read block 256 len 4096
99642024-02-04T14:04:41.800Z05/30 Write block 268 len 24576 data: 9 8 7 8 4 5
99652024-02-04T14:04:41.803Z06/30 Write block 102 len 8192 data: 5 5
99662024-02-04T14:04:41.803Z07/30 Flush
99672024-02-04T14:04:41.806Z08/30 Write block 224 len 4096 data: 5
99682024-02-04T14:04:41.806Z09/30 Write block 528 len 16384 data: 6 5 4 3
99692024-02-04T14:04:41.809Z10/30 Write block 99 len 16384 data: 7 5 6 6
99702024-02-04T14:04:41.809Z11/30 Write block 214 len 4096 data: 7
99712024-02-04T14:04:41.809Z12/30 Read block 138 len 36864
99722024-02-04T14:04:41.812Z13/30 Flush
99732024-02-04T14:04:41.815Z14/30 Write block 310 len 4096 data: 7
99742024-02-04T14:04:41.815Z15/30 Write block 521 len 24576 data: 4 5 4 5 4 6
99752024-02-04T14:04:41.817Z16/30 Write block 245 len 8192 data: 5 5
99762024-02-04T14:04:41.817Z17/30 Write block 266 len 20480 data: 9 8 10 9 8
99772024-02-04T14:04:41.818Z18/30 Read block 453 len 4096
99782024-02-04T14:04:41.820Z19/30 Read block 502 len 12288
99792024-02-04T14:04:41.820Z20/30 Write block 391 len 40960 data: 10 11 13 12 11 10 12 10 10 11
99802024-02-04T14:04:41.823Z21/30 Read block 560 len 4096
99812024-02-04T14:04:41.823Z22/30 Write block 191 len 24576 data: 7 7 5 3 3 4
99822024-02-04T14:04:41.824Z23/30 Read block 315 len 28672
99832024-02-04T14:04:41.829Z24/30 Read block 232 len 40960
99842024-02-04T14:04:41.832Z25/30 Read block 455 len 40960
99852024-02-04T14:04:41.832Z26/30 Read block 178 len 20480
99862024-02-04T14:04:41.832Z27/30 Read block 516 len 28672
99872024-02-04T14:04:41.832Z28/30 Read block 133 len 8192
99882024-02-04T14:04:41.835Z29/30 Read block 326 len 4096
99892024-02-04T14:04:41.835Z30/30 Write block 141 len 24576 data: 6 3 4 7 8 8
99902024-02-04T14:04:41.835Z----------------------------------------------------------------
99912024-02-04T14:04:41.835Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
99922024-02-04T14:04:41.835ZStates: Active Active Active
99932024-02-04T14:04:41.835ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99942024-02-04T14:04:41.835Z 1 Acked 1000 Write 10 Done Done Done false
99952024-02-04T14:04:41.835Z 2 Acked 1001 Read 10 Done Done Done false
99962024-02-04T14:04:41.836Z 3 Acked 1002 Read 10 Done Done Sent false
99972024-02-04T14:04:41.836Z 4 Acked 1003 Read 1 Done Done Sent false
99982024-02-04T14:04:41.836Z 5 Acked 1004 Write 6 Done Done Sent false
99992024-02-04T14:04:41.836Z 6 Acked 1005 Write 2 Done Done Sent false
100002024-02-04T14:04:41.836Z 7 Acked 1006 Flush 0 Done Done Sent false
100012024-02-04T14:04:41.836Z 8 Acked 1007 Write 1 Done Done Sent false
100022024-02-04T14:04:41.836Z 9 Acked 1008 Write 4 Done Done Sent false
100032024-02-04T14:04:41.836Z 10 Acked 1009 Write 4 Done Done Sent false
100042024-02-04T14:04:41.836Z 11 Acked 1010 Write 1 Done Done Sent false
100052024-02-04T14:04:41.836Z 12 Acked 1011 Read 9 Done Done Sent false
100062024-02-04T14:04:41.836Z 13 Acked 1012 Flush 0 Done Done Sent false
100072024-02-04T14:04:41.836Z 14 Acked 1013 Write 1 Done Done Sent false
100082024-02-04T14:04:41.836Z 15 Acked 1014 Write 6 Done Done Sent false
100092024-02-04T14:04:41.836Z 16 Acked 1015 Write 2 Done Done Sent false
100102024-02-04T14:04:41.836Z 17 Acked 1016 Write 5 Done Done Sent false
100112024-02-04T14:04:41.836Z 18 Acked 1017 Read 1 Done Done Sent false
100122024-02-04T14:04:41.836Z 19 Acked 1018 Read 3 Done Done Sent false
100132024-02-04T14:04:41.836Z 20 Acked 1019 Write 10 Done Done Sent false
100142024-02-04T14:04:41.836Z 21 Acked 1020 Read 1 Done Done Sent false
100152024-02-04T14:04:41.836Z 22 Acked 1021 Write 6 Done Done Sent false
100162024-02-04T14:04:41.836Z 23 Acked 1022 Read 7 Done Done Sent false
100172024-02-04T14:04:41.836Z 24 Acked 1023 Read 10 Done Done Sent false
100182024-02-04T14:04:41.836Z 25 Acked 1024 Read 10 Done Done Sent false
100192024-02-04T14:04:41.836Z 26 Acked 1025 Read 5 Done Done Sent false
100202024-02-04T14:04:41.837Z 27 Acked 1026 Read 7 Done Done Sent false
100212024-02-04T14:04:41.837Z 28 Acked 1027 Read 2 Done Done Sent false
100222024-02-04T14:04:41.837Z 29 Acked 1028 Read 1 Done Done Sent false
100232024-02-04T14:04:41.837Z 30 Acked 1029 Write 6 Sent Sent Sent false
100242024-02-04T14:04:41.837Z STATES DS:0 DS:1 DS:2 TOTAL
100252024-02-04T14:04:41.837Z New 0 0 0 0
100262024-02-04T14:04:41.837Z Sent 1 1 28 30
100272024-02-04T14:04:41.837Z Done 29 29 2 60
100282024-02-04T14:04:41.837Z Skipped 0 0 0 0
100292024-02-04T14:04:41.837Z Error 0 0 0 0
100302024-02-04T14:04:41.837ZLast Flush: 1012 1012 0
100312024-02-04T14:04:41.837ZDownstairs last five completed:
100322024-02-04T14:04:41.837ZUpstairs last five completed: 30 29 28 27 26
100332024-02-04T14:04:41.840Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:40.868534366Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1497}
100342024-02-04T14:04:41.840ZWrote out file "/tmp/test_repair_verify.data"
100352024-02-04T14:04:41.843Z
100362024-02-04T14:04:41.847ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100372024-02-04T14:04:41.873ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100382024-02-04T14:04:41.873Z 0 000-019 87 87 87 139 139 139 F F F
100392024-02-04T14:04:41.873Z 1 020-039 91 91 91 148 148 148 F F F
100402024-02-04T14:04:41.873Z 2 040-059 88 88 88 142 142 142 F F F
100412024-02-04T14:04:41.873Z 3 060-079 82 82 82 129 129 129 F F F
100422024-02-04T14:04:41.873Z 4 080-099 92 92 90 150 150 145 F F F <---
100432024-02-04T14:04:41.874Z 5 100-119 92 92 88 150 150 142 F F F <---
100442024-02-04T14:04:41.877Z 6 120-139 88 88 88 142 142 142 F F F
100452024-02-04T14:04:41.877Z 7 140-159 90 90 90 145 145 145 T T F <---
100462024-02-04T14:04:41.877Z 8 160-179 88 88 88 143 143 143 F F F
100472024-02-04T14:04:41.877Z 9 180-199 88 88 88 142 142 142 T T F <---
100482024-02-04T14:04:41.877Z 10 200-219 92 92 89 150 150 144 F F F <---
100492024-02-04T14:04:41.877Z 11 220-239 92 92 90 150 150 145 F F F <---
100502024-02-04T14:04:41.877Z 12 240-259 84 84 84 132 132 132 T T F <---
100512024-02-04T14:04:41.877Z 13 260-279 92 92 88 149 149 142 T T F <---
100522024-02-04T14:04:41.877Z 14 280-299 88 88 88 142 142 142 F F F
100532024-02-04T14:04:41.877Z 15 300-319 74 74 74 119 119 119 T T F <---
100542024-02-04T14:04:41.877Z 16 320-339 86 86 86 137 137 137 F F F
100552024-02-04T14:04:41.877Z 17 340-359 88 88 88 142 142 142 F F F
100562024-02-04T14:04:41.877Z 18 360-379 90 90 90 145 145 145 F F F
100572024-02-04T14:04:41.877Z 19 380-399 90 90 90 145 145 145 T T F <---
100582024-02-04T14:04:41.877Z 20 400-419 92 92 90 149 149 147 T T T <---
100592024-02-04T14:04:41.878Z 21 420-439 86 86 86 138 138 138 F F F
100602024-02-04T14:04:41.878Z 22 440-459 89 89 89 144 144 144 F F F
100612024-02-04T14:04:41.878Z 23 460-479 85 85 85 135 135 135 F F F
100622024-02-04T14:04:41.878Z 24 480-499 90 90 90 146 146 146 F F F
100632024-02-04T14:04:41.878Z 25 500-519 77 77 77 122 122 122 F F F
100642024-02-04T14:04:41.878Z 26 520-539 92 92 84 150 150 132 T T F <---
100652024-02-04T14:04:41.878Z 27 540-559 83 83 83 131 131 131 F F F
100662024-02-04T14:04:41.878Z 28 560-579 84 84 84 132 132 132 F F F
100672024-02-04T14:04:41.878Z 29 580-599 90 90 90 146 146 146 F F F
100682024-02-04T14:04:41.878ZMax gen: 92, Max flush: 150
100692024-02-04T14:04:41.878ZError: Difference in extent metadata found!
100702024-02-04T14:04:41.881ZA Difference in extent metadata is expected here
100712024-02-04T14:04:41.881ZOn loop 46
100722024-02-04T14:04:41.881Z
100732024-02-04T14:04:41.884ZVerifying data now
100742024-02-04T14:04:46.940ZNo change after verify
100752024-02-04T14:04:46.940ZLoop: 46 Downstairs dump after verify (and repair):
100762024-02-04T14:04:46.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100772024-02-04T14:04:46.962Z 0 000-019 87 87 87 139 139 139 F F F
100782024-02-04T14:04:46.962Z 1 020-039 91 91 91 148 148 148 F F F
100792024-02-04T14:04:46.962Z 2 040-059 88 88 88 142 142 142 F F F
100802024-02-04T14:04:46.962Z 3 060-079 82 82 82 129 129 129 F F F
100812024-02-04T14:04:46.962Z 4 080-099 92 92 92 150 150 150 F F F
100822024-02-04T14:04:46.963Z 5 100-119 92 92 92 150 150 150 F F F
100832024-02-04T14:04:46.966Z 6 120-139 88 88 88 142 142 142 F F F
100842024-02-04T14:04:46.966Z 7 140-159 93 93 93 151 151 151 F F F
100852024-02-04T14:04:46.966Z 8 160-179 88 88 88 143 143 143 F F F
100862024-02-04T14:04:46.966Z 9 180-199 93 93 93 151 151 151 F F F
100872024-02-04T14:04:46.966Z 10 200-219 92 92 92 150 150 150 F F F
100882024-02-04T14:04:46.966Z 11 220-239 92 92 92 150 150 150 F F F
100892024-02-04T14:04:46.966Z 12 240-259 93 93 93 151 151 151 F F F
100902024-02-04T14:04:46.966Z 13 260-279 93 93 93 151 151 151 F F F
100912024-02-04T14:04:46.966Z 14 280-299 88 88 88 142 142 142 F F F
100922024-02-04T14:04:46.966Z 15 300-319 93 93 93 151 151 151 F F F
100932024-02-04T14:04:46.966Z 16 320-339 86 86 86 137 137 137 F F F
100942024-02-04T14:04:46.966Z 17 340-359 88 88 88 142 142 142 F F F
100952024-02-04T14:04:46.966Z 18 360-379 90 90 90 145 145 145 F F F
100962024-02-04T14:04:46.966Z 19 380-399 93 93 93 151 151 151 F F F
100972024-02-04T14:04:46.966Z 20 400-419 93 93 93 151 151 151 F F F
100982024-02-04T14:04:46.966Z 21 420-439 86 86 86 138 138 138 F F F
100992024-02-04T14:04:46.966Z 22 440-459 89 89 89 144 144 144 F F F
101002024-02-04T14:04:46.966Z 23 460-479 85 85 85 135 135 135 F F F
101012024-02-04T14:04:46.967Z 24 480-499 90 90 90 146 146 146 F F F
101022024-02-04T14:04:46.967Z 25 500-519 77 77 77 122 122 122 F F F
101032024-02-04T14:04:46.967Z 26 520-539 93 93 93 151 151 151 F F F
101042024-02-04T14:04:46.967Z 27 540-559 83 83 83 131 131 131 F F F
101052024-02-04T14:04:46.967Z 28 560-579 84 84 84 132 132 132 F F F
101062024-02-04T14:04:46.967Z 29 580-599 90 90 90 146 146 146 F F F
101072024-02-04T14:04:46.967ZMax gen: 93, Max flush: 151
101082024-02-04T14:04:46.970Z
101092024-02-04T14:04:46.970ZBegin loop 47
101102024-02-04T14:04:46.970ZDownstairs to restart: 1
101112024-02-04T14:04:47.004Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.032688726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505}
101122024-02-04T14:04:47.005Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033158114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505}
101132024-02-04T14:04:47.010Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.03320714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505}
101142024-02-04T14:04:47.010Z{"msg":"Crucible a5649e2a-80ef-4403-8515-1028ee3a34e7 has session id: 0a5e817c-d624-4ebe-9bf0-e44592fac23c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033272156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101152024-02-04T14:04:47.010Z{"msg":"Upstairs opts: Upstairs UUID: a5649e2a-80ef-4403-8515-1028ee3a34e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033325665Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101162024-02-04T14:04:47.010Z{"Crucible runtime is spawned
101172024-02-04T14:04:47.010Zmsg":"Crucible stats registered with UUID: a5649e2a-80ef-4403-8515-1028ee3a34e7","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033533017Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101182024-02-04T14:04:47.010Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033634714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505}
101192024-02-04T14:04:47.010Z{"msg":"a5649e2a-80ef-4403-8515-1028ee3a34e7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033735577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101202024-02-04T14:04:47.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033843674Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101212024-02-04T14:04:47.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033872695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101222024-02-04T14:04:47.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033895849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101232024-02-04T14:04:47.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.033972851Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101242024-02-04T14:04:47.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.03400806Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101252024-02-04T14:04:47.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.034032383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101262024-02-04T14:04:47.010Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.034099086Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101272024-02-04T14:04:47.010Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.034466729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101282024-02-04T14:04:47.010Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.034608087Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101292024-02-04T14:04:47.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.034693977Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101302024-02-04T14:04:47.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.034769184Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101312024-02-04T14:04:47.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.034817379Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101322024-02-04T14:04:47.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.035671403Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101332024-02-04T14:04:47.011Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.035926536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101342024-02-04T14:04:47.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.036036161Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101352024-02-04T14:04:47.011Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.036086371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101362024-02-04T14:04:47.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.036242193Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101372024-02-04T14:04:47.011Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.03631438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101382024-02-04T14:04:47.011Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.036959962Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101392024-02-04T14:04:47.011Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.037314777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101402024-02-04T14:04:47.011Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.037621165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101412024-02-04T14:04:47.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038289112Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101422024-02-04T14:04:47.011Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038377022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101432024-02-04T14:04:47.011Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038423001Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101442024-02-04T14:04:47.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038513445Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101452024-02-04T14:04:47.011Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038589541Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101462024-02-04T14:04:47.011Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038618196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101472024-02-04T14:04:47.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038674069Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101482024-02-04T14:04:47.011Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038705823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101492024-02-04T14:04:47.011Z{"msg":"[0]R flush_numbers[0..12]: [139, 148, 142, 129, 150, 150, 142, 151, 143, 151, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038762315Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101502024-02-04T14:04:47.011Z{"msg":"[0]R generation[0..12]: [87, 91, 88, 82, 92, 92, 88, 93, 88, 93, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038794867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101512024-02-04T14:04:47.011Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
101522024-02-04T14:04:47.011Z,"time":"2024-02-04T14:04:46.038889768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101532024-02-04T14:04:47.011Z{"msg":"[1]R flush_numbers[0..12]: [139, 148, 142, 129, 150, 150, 142, 151, 143, 151, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038960152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101542024-02-04T14:04:47.011Z{"msg":"[1]R generation[0..12]: [87, 91, 88, 82, 92, 92, 88, 93, 88, 93, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.038987563Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101552024-02-04T14:04:47.012Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.03902935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101562024-02-04T14:04:47.012Z{"msg":"[2]R flush_numbers[0..12]: [139, 148, 142, 129, 150, 150, 142, 151, 143, 151, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.03908428Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101572024-02-04T14:04:47.012Z{"msg":"[2]R generation[0..12]: [87, 91, 88, 82, 92, 92, 88, 93, 88, 93, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039115103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101582024-02-04T14:04:47.012Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039172064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101592024-02-04T14:04:47.012Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.03919911Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101602024-02-04T14:04:47.012Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039259379Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101612024-02-04T14:04:47.012Z{"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039289718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101622024-02-04T14:04:47.012Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039346997Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101632024-02-04T14:04:47.012Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039372402Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101642024-02-04T14:04:47.012Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.03943134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101652024-02-04T14:04:47.012Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039462317Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101662024-02-04T14:04:47.012Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039517422Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101672024-02-04T14:04:47.012Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039543988Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101682024-02-04T14:04:47.012Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039600499Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101692024-02-04T14:04:47.012Z{"msg":"a5649e2a-80ef-4403-8515-1028ee3a34e7 is now active with session: 0a5e817c-d624-4ebe-9bf0-e44592fac23c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039625466Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101702024-02-04T14:04:47.012Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039682421Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505,"session_id":"0a5e817c-d624-4ebe-9bf0-e44592fac23c"}
101712024-02-04T14:04:47.012Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:46.039710547Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505}
101722024-02-04T14:04:47.012ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101732024-02-04T14:04:47.012ZLoading write count information from file "/tmp/test_repair_verify.data"
101742024-02-04T14:04:47.012ZRun Repair workload
101752024-02-04T14:04:47.012Z01/30 Write block 496 len 36864 data: 12 10 11 12 13 13 10 10 11
101762024-02-04T14:04:47.016Z02/30 Read block 227 len 36864
101772024-02-04T14:04:47.016Z03/30 Read block 287 len 32768
101782024-02-04T14:04:47.016Z04/30 Read block 90 len 36864
101792024-02-04T14:04:47.019Z05/30 Read block 319 len 16384
101802024-02-04T14:04:47.019Z06/30 Read block 160 len 20480
101812024-02-04T14:04:47.021Z07/30 Read block 413 len 40960
101822024-02-04T14:04:47.021Z08/30 Write block 305 len 8192 data: 8 7
101832024-02-04T14:04:47.024Z09/30 Write block 543 len 28672 data: 8 6 6 4 4 5 6
101842024-02-04T14:04:47.024Z10/30 Read block 583 len 20480
101852024-02-04T14:04:47.027Z11/30 Read block 364 len 16384
101862024-02-04T14:04:47.027Z12/30 Read block 315 len 20480
101872024-02-04T14:04:47.027Z13/30 Read block 415 len 4096
101882024-02-04T14:04:47.030Z14/30 Read block 277 len 24576
101892024-02-04T14:04:47.030Z15/30 Flush
101902024-02-04T14:04:47.034Z16/30 Write block 448 len 32768 data: 8 7 8 9 11 10 12 12
101912024-02-04T14:04:47.036Z17/30 Write block 181 len 12288 data: 10 9 11
101922024-02-04T14:04:47.036Z18/30 Write block 283 len 28672 data: 9 14 12 10 11 11 10
101932024-02-04T14:04:47.037Z19/30 Read block 0 len 32768
101942024-02-04T14:04:47.039Z20/30 Flush
101952024-02-04T14:04:47.042Z21/30 Write block 248 len 36864 data: 10 13 14 13 13 9 10 11 10
101962024-02-04T14:04:47.045Z22/30 Read block 329 len 40960
101972024-02-04T14:04:47.060Z23/30 Read block 201 len 8192
101982024-02-04T14:04:47.060Z24/30 Flush
101992024-02-04T14:04:47.060Z25/30 Flush
102002024-02-04T14:04:47.060Z26/30 Write block 80 len 20480 data: 9 10 11 9 9
102012024-02-04T14:04:47.060Z27/30 Write block 163 len 4096 data: 8
102022024-02-04T14:04:47.060Z28/30 Read block 318 len 12288
102032024-02-04T14:04:47.060Z29/30 Read block 578 len 20480
102042024-02-04T14:04:47.061Z30/30 Read block 123 len 32768
102052024-02-04T14:04:47.061Z----------------------------------------------------------------
102062024-02-04T14:04:47.061Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
102072024-02-04T14:04:47.061ZStates: Active Active Active
102082024-02-04T14:04:47.061ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102092024-02-04T14:04:47.061Z 1 Acked 1000 Write 9 Done Done Done false
102102024-02-04T14:04:47.061Z 2 Acked 1001 Read 9 Done Done Done false
102112024-02-04T14:04:47.061Z 3 Acked 1002 Read 8 Done Done Done false
102122024-02-04T14:04:47.061Z 4 Acked 1003 Read 9 Done Sent Done false
102132024-02-04T14:04:47.061Z 5 Acked 1004 Read 4 Done Sent Done false
102142024-02-04T14:04:47.061Z 6 Acked 1005 Read 5 Done Sent Done false
102152024-02-04T14:04:47.061Z 7 Acked 1006 Read 10 Done Sent Done false
102162024-02-04T14:04:47.061Z 8 Acked 1007 Write 2 Done Sent Done false
102172024-02-04T14:04:47.061Z 9 Acked 1008 Write 7 Done Sent Done false
102182024-02-04T14:04:47.061Z 10 Acked 1009 Read 5 Done Sent Done false
102192024-02-04T14:04:47.061Z 11 Acked 1010 Read 4 Done Sent Done false
102202024-02-04T14:04:47.061Z 12 Acked 1011 Read 5 Done Sent Done false
102212024-02-04T14:04:47.061Z 13 Acked 1012 Read 1 Done Sent Done false
102222024-02-04T14:04:47.061Z 14 Acked 1013 Read 6 Done Sent Done false
102232024-02-04T14:04:47.061Z 15 Acked 1014 Flush 0 Done Sent Done false
102242024-02-04T14:04:47.061Z 16 Acked 1015 Write 8 Done Sent Done false
102252024-02-04T14:04:47.061Z 17 Acked 1016 Write 3 Done Sent Done false
102262024-02-04T14:04:47.061Z 18 Acked 1017 Write 7 Done Sent Done false
102272024-02-04T14:04:47.061Z 19 Acked 1018 Read 8 Done Sent Done false
102282024-02-04T14:04:47.061Z 20 Acked 1019 Flush 0 Done Sent Done false
102292024-02-04T14:04:47.062Z 21 Acked 1020 Write 9 Done Sent Done false
102302024-02-04T14:04:47.062Z 22 Acked 1021 Read 10 Done Sent Done false
102312024-02-04T14:04:47.062Z 23 Acked 1022 Read 2 Done Sent Done false
102322024-02-04T14:04:47.062Z 24 Acked 1023 Flush 0 Done Sent Done false
102332024-02-04T14:04:47.062Z 25 Acked 1024 Flush 0 Done Sent Done false
102342024-02-04T14:04:47.062Z 26 Acked 1025 Write 5 Done Sent Done false
102352024-02-04T14:04:47.062Z 27 Acked 1026 Write 1 Done Sent Done false
102362024-02-04T14:04:47.062Z 28 Acked 1027 Read 3 Done Sent Done false
102372024-02-04T14:04:47.062Z 29 Acked 1028 Read 5 Done Sent Done false
102382024-02-04T14:04:47.062Z 30 Acked 1029 Read 8 Sent Sent Done false
102392024-02-04T14:04:47.062Z STATES DS:0 DS:1 DS:2 TOTAL
102402024-02-04T14:04:47.062Z New 0 0 0 0
102412024-02-04T14:04:47.062Z Sent 1 27 0 28
102422024-02-04T14:04:47.062Z Done 29 3 30 62
102432024-02-04T14:04:47.062Z Skipped 0 0 0 0
102442024-02-04T14:04:47.062Z Error 0 0 0 0
102452024-02-04T14:04:47.062ZLast Flush: 1024 0 1024
102462024-02-04T14:04:47.062ZDownstairs last five completed:
102472024-02-04T14:04:47.062ZUpstairs last five completed: 30 29 28 27 26
102482024-02-04T14:04:47.062Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:46.089497666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1505}
102492024-02-04T14:04:47.062ZWrote out file "/tmp/test_repair_verify.data"
102502024-02-04T14:04:47.067Z
102512024-02-04T14:04:47.067ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102522024-02-04T14:04:47.090ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102532024-02-04T14:04:47.090Z 0 000-019 87 87 87 139 139 139 F F F
102542024-02-04T14:04:47.090Z 1 020-039 91 91 91 148 148 148 F F F
102552024-02-04T14:04:47.090Z 2 040-059 88 88 88 142 142 142 F F F
102562024-02-04T14:04:47.090Z 3 060-079 82 82 82 129 129 129 F F F
102572024-02-04T14:04:47.090Z 4 080-099 92 92 92 150 150 150 T F T <---
102582024-02-04T14:04:47.090Z 5 100-119 92 92 92 150 150 150 F F F
102592024-02-04T14:04:47.093Z 6 120-139 88 88 88 142 142 142 F F F
102602024-02-04T14:04:47.093Z 7 140-159 93 93 93 151 151 151 F F F
102612024-02-04T14:04:47.093Z 8 160-179 88 88 88 143 143 143 T F T <---
102622024-02-04T14:04:47.093Z 9 180-199 94 93 94 153 151 153 F F F <---
102632024-02-04T14:04:47.093Z 10 200-219 92 92 92 150 150 150 F F F
102642024-02-04T14:04:47.093Z 11 220-239 92 92 92 150 150 150 F F F
102652024-02-04T14:04:47.093Z 12 240-259 94 93 94 154 151 154 F F F <---
102662024-02-04T14:04:47.093Z 13 260-279 93 93 93 151 151 151 F F F
102672024-02-04T14:04:47.094Z 14 280-299 94 88 94 153 142 153 F F F <---
102682024-02-04T14:04:47.094Z 15 300-319 94 93 94 152 151 152 F F F <---
102692024-02-04T14:04:47.094Z 16 320-339 86 86 86 137 137 137 F F F
102702024-02-04T14:04:47.094Z 17 340-359 88 88 88 142 142 142 F F F
102712024-02-04T14:04:47.094Z 18 360-379 90 90 90 145 145 145 F F F
102722024-02-04T14:04:47.094Z 19 380-399 93 93 93 151 151 151 F F F
102732024-02-04T14:04:47.094Z 20 400-419 93 93 93 151 151 151 F F F
102742024-02-04T14:04:47.094Z 21 420-439 86 86 86 138 138 138 F F F
102752024-02-04T14:04:47.094Z 22 440-459 94 89 94 153 144 153 F F F <---
102762024-02-04T14:04:47.094Z 23 460-479 85 85 85 135 135 135 F F F
102772024-02-04T14:04:47.094Z 24 480-499 94 90 94 152 146 152 F T F <---
102782024-02-04T14:04:47.094Z 25 500-519 94 77 94 152 122 152 F T F <---
102792024-02-04T14:04:47.094Z 26 520-539 93 93 93 151 151 151 F F F
102802024-02-04T14:04:47.094Z 27 540-559 94 83 94 152 131 152 F F F <---
102812024-02-04T14:04:47.094Z 28 560-579 84 84 84 132 132 132 F F F
102822024-02-04T14:04:47.094Z 29 580-599 90 90 90 146 146 146 F F F
102832024-02-04T14:04:47.094ZMax gen: 94, Max flush: 154
102842024-02-04T14:04:47.094ZError: Difference in extent metadata found!
102852024-02-04T14:04:47.097ZA Difference in extent metadata is expected here
102862024-02-04T14:04:47.097ZOn loop 47
102872024-02-04T14:04:47.097Z
102882024-02-04T14:04:47.100ZVerifying data now
102892024-02-04T14:04:52.996ZNo change after verify
102902024-02-04T14:04:52.996ZLoop: 47 Downstairs dump after verify (and repair):
102912024-02-04T14:04:53.017ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102922024-02-04T14:04:53.017Z 0 000-019 87 87 87 139 139 139 F F F
102932024-02-04T14:04:53.018Z 1 020-039 91 91 91 148 148 148 F F F
102942024-02-04T14:04:53.018Z 2 040-059 88 88 88 142 142 142 F F F
102952024-02-04T14:04:53.018Z 3 060-079 82 82 82 129 129 129 F F F
102962024-02-04T14:04:53.018Z 4 080-099 95 95 95 155 155 155 F F F
102972024-02-04T14:04:53.021Z 5 100-119 92 92 92 150 150 150 F F F
102982024-02-04T14:04:53.021Z 6 120-139 88 88 88 142 142 142 F F F
102992024-02-04T14:04:53.022Z 7 140-159 93 93 93 151 151 151 F F F
103002024-02-04T14:04:53.022Z 8 160-179 95 95 95 155 155 155 F F F
103012024-02-04T14:04:53.022Z 9 180-199 94 94 94 153 153 153 F F F
103022024-02-04T14:04:53.022Z 10 200-219 92 92 92 150 150 150 F F F
103032024-02-04T14:04:53.022Z 11 220-239 92 92 92 150 150 150 F F F
103042024-02-04T14:04:53.022Z 12 240-259 94 94 94 154 154 154 F F F
103052024-02-04T14:04:53.022Z 13 260-279 93 93 93 151 151 151 F F F
103062024-02-04T14:04:53.022Z 14 280-299 94 94 94 153 153 153 F F F
103072024-02-04T14:04:53.022Z 15 300-319 94 94 94 152 152 152 F F F
103082024-02-04T14:04:53.022Z 16 320-339 86 86 86 137 137 137 F F F
103092024-02-04T14:04:53.022Z 17 340-359 88 88 88 142 142 142 F F F
103102024-02-04T14:04:53.022Z 18 360-379 90 90 90 145 145 145 F F F
103112024-02-04T14:04:53.022Z 19 380-399 93 93 93 151 151 151 F F F
103122024-02-04T14:04:53.023Z 20 400-419 93 93 93 151 151 151 F F F
103132024-02-04T14:04:53.023Z 21 420-439 86 86 86 138 138 138 F F F
103142024-02-04T14:04:53.023Z 22 440-459 94 94 94 153 153 153 F F F
103152024-02-04T14:04:53.023Z 23 460-479 85 85 85 135 135 135 F F F
103162024-02-04T14:04:53.023Z 24 480-499 94 94 94 152 152 152 F F F
103172024-02-04T14:04:53.023Z 25 500-519 94 94 94 152 152 152 F F F
103182024-02-04T14:04:53.023Z 26 520-539 93 93 93 151 151 151 F F F
103192024-02-04T14:04:53.023Z 27 540-559 94 94 94 152 152 152 F F F
103202024-02-04T14:04:53.023Z 28 560-579 84 84 84 132 132 132 F F F
103212024-02-04T14:04:53.023Z 29 580-599 90 90 90 146 146 146 F F F
103222024-02-04T14:04:53.023ZMax gen: 95, Max flush: 155
103232024-02-04T14:04:53.027Z
103242024-02-04T14:04:53.027ZBegin loop 48
103252024-02-04T14:04:53.027ZDownstairs to restart: 0
103262024-02-04T14:04:53.052Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.084517684Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513}
103272024-02-04T14:04:53.052Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.084987017Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513}
103282024-02-04T14:04:53.056Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085049384Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513}
103292024-02-04T14:04:53.056Z{"msg":"Crucible 70caf5eb-a376-479d-9a5f-1556323c168d has session id: 883f74e5-8253-455b-a0c6-2e0c0f011103","v":0,"name":"crucible","level":30Crucible runtime is spawned
103302024-02-04T14:04:53.056Z,"time":"2024-02-04T14:04:52.085197807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103312024-02-04T14:04:53.056Z{"msg":"Upstairs opts: Upstairs UUID: 70caf5eb-a376-479d-9a5f-1556323c168d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085348496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103322024-02-04T14:04:53.056Z{"msg":"Crucible stats registered with UUID: 70caf5eb-a376-479d-9a5f-1556323c168d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085391159Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103332024-02-04T14:04:53.056Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085438063Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513}
103342024-02-04T14:04:53.056Z{"msg":"70caf5eb-a376-479d-9a5f-1556323c168d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085481076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103352024-02-04T14:04:53.056Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.08552394Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103362024-02-04T14:04:53.056Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.0855578Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103372024-02-04T14:04:53.056Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085626014Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103382024-02-04T14:04:53.056Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085674116Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103392024-02-04T14:04:53.057Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085716894Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103402024-02-04T14:04:53.057Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.085925236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103412024-02-04T14:04:53.057Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.086030634Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"io task","client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103422024-02-04T14:04:53.057Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.086133695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"io task","client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103432024-02-04T14:04:53.057Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.08621542Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"io task","client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103442024-02-04T14:04:53.057Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.086284968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"io task","client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103452024-02-04T14:04:53.057Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.086313272Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"io task","client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103462024-02-04T14:04:53.057Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.086382859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"io task","client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103472024-02-04T14:04:53.057Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.087312535Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103482024-02-04T14:04:53.057Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.087461309Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103492024-02-04T14:04:53.057Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.087587756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103502024-02-04T14:04:53.057Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.087634307Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103512024-02-04T14:04:53.057Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.088380276Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103522024-02-04T14:04:53.057Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.08844539Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103532024-02-04T14:04:53.057Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.088491424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103542024-02-04T14:04:53.062Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.088688973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103552024-02-04T14:04:53.062Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.089445928Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103562024-02-04T14:04:53.062Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.089533462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103572024-02-04T14:04:53.062Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.089584465Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103582024-02-04T14:04:53.062Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.089627567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103592024-02-04T14:04:53.062Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.089692834Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103602024-02-04T14:04:53.062Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.089720114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103612024-02-04T14:04:53.062Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.089875463Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103622024-02-04T14:04:53.062Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.090616461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103632024-02-04T14:04:53.062Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.090688235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103642024-02-04T14:04:53.062Z{"msg":"[0]R flush_numbers[0..12]: [139, 148, 142, 129, 155, 150, 142, 151, 155, 153, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.090737106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103652024-02-04T14:04:53.062Z{"msg":"[0]R generation[0..12]: [87, 91, 88, 82, 95, 92, 88, 93, 95, 94, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.090797403Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103662024-02-04T14:04:53.062Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.090844822Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103672024-02-04T14:04:53.062ZWait for a query_work_queue command to finish before sending IO
103682024-02-04T14:04:53.063Z{"msg":"[1]R flush_numbers[0..12]: [139, 148, 142, 129, 155, 150, 142, 151, 155, 153, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.090903882Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103692024-02-04T14:04:53.063Z{"msg":"[1]R generation[0..12]: [87, 91, 88, 82, 95, 92, 88, 93, 95, 94, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.090949282Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103702024-02-04T14:04:53.063Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.0910446Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103712024-02-04T14:04:53.063Z{"msg":"[2]R flush_numbers[0..12]: [139, 148, 142, 129, 155, 150, 142, 151, 155, 153, 150, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091116684Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103722024-02-04T14:04:53.063Z{"msg":"[2]R generation[0..12]: [87, 91, 88, 82, 95, 92, 88, 93, 95, 94, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.09114431Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103732024-02-04T14:04:53.063Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091169054Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103742024-02-04T14:04:53.063Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091191167Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103752024-02-04T14:04:53.063Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091213616Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103762024-02-04T14:04:53.063Z{"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091234973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103772024-02-04T14:04:53.063Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091257167Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103782024-02-04T14:04:53.063Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091279734Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103792024-02-04T14:04:53.063Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091325705Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103802024-02-04T14:04:53.063Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.09135265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103812024-02-04T14:04:53.063Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091376384Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103822024-02-04T14:04:53.063Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091399019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103832024-02-04T14:04:53.063Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091420117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103842024-02-04T14:04:53.063Z{"msg":"70caf5eb-a376-479d-9a5f-1556323c168d is now active with session: 883f74e5-8253-455b-a0c6-2e0c0f011103","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091466988Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103852024-02-04T14:04:53.064Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091488852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513,"session_id":"883f74e5-8253-455b-a0c6-2e0c0f011103"}
103862024-02-04T14:04:53.064Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:52.091509956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513}
103872024-02-04T14:04:53.064ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103882024-02-04T14:04:53.064ZLoading write count information from file "/tmp/test_repair_verify.data"
103892024-02-04T14:04:53.064ZRun Repair workload
103902024-02-04T14:04:53.064Z01/30 Write block 443 len 4096 data: 7
103912024-02-04T14:04:53.067Z02/30 Read block 484 len 24576
103922024-02-04T14:04:53.067Z03/30 Write block 48 len 4096 data: 7
103932024-02-04T14:04:53.067Z04/30 Write block 521 len 32768 data: 5 6 5 6 5 7 6 7
103942024-02-04T14:04:53.067Z05/30 Flush
103952024-02-04T14:04:53.073Z06/30 Write block 506 len 40960 data: 12 12 11 11 9 8 5 5 7 10
103962024-02-04T14:04:53.073Z07/30 Read block 313 len 36864
103972024-02-04T14:04:53.076Z08/30 Flush
103982024-02-04T14:04:53.078Z09/30 Write block 486 len 32768 data: 6 6 7 10 10 11 13 13
103992024-02-04T14:04:53.078Z10/30 Read block 345 len 20480
104002024-02-04T14:04:53.081Z11/30 Read block 431 len 28672
104012024-02-04T14:04:53.081Z12/30 Write block 440 len 40960 data: 6 7 7 8 8 8 9 9 9 8
104022024-02-04T14:04:53.081Z13/30 Read block 590 len 32768
104032024-02-04T14:04:53.084Z14/30 Read block 101 len 28672
104042024-02-04T14:04:53.085Z15/30 Write block 380 len 12288 data: 10 8 5
104052024-02-04T14:04:53.089Z16/30 Write block 564 len 32768 data: 9 10 10 8 6 6 5 5
104062024-02-04T14:04:53.089Z17/30 Write block 212 len 40960 data: 6 6 8 7 9 10 7 7 8 9
104072024-02-04T14:04:53.089Z18/30 Read block 72 len 40960
104082024-02-04T14:04:53.092Z19/30 Read block 311 len 24576
104092024-02-04T14:04:53.092Z20/30 Read block 460 len 32768
104102024-02-04T14:04:53.095Z21/30 Read block 383 len 36864
104112024-02-04T14:04:53.095Z22/30 Read block 413 len 16384
104122024-02-04T14:04:53.098Z23/30 Write block 93 len 12288 data: 3 3 6
104132024-02-04T14:04:53.099Z24/30 Write block 432 len 28672 data: 7 7 8 8 7 6 6
104142024-02-04T14:04:53.102Z25/30 Read block 246 len 28672
104152024-02-04T14:04:53.102Z26/30 Write block 77 len 8192 data: 10 9
104162024-02-04T14:04:53.105Z27/30 Write block 433 len 16384 data: 8 9 9 8
104172024-02-04T14:04:53.105Z28/30 Read block 115 len 32768
104182024-02-04T14:04:53.109Z29/30 Write block 317 len 8192 data: 5 6
104192024-02-04T14:04:53.109Z30/30 Read block 203 len 20480
104202024-02-04T14:04:53.109Z----------------------------------------------------------------
104212024-02-04T14:04:53.109Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
104222024-02-04T14:04:53.109ZStates: Active Active Active
104232024-02-04T14:04:53.110ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104242024-02-04T14:04:53.110Z 1 Acked 1000 Write 1 Sent Done Done false
104252024-02-04T14:04:53.110Z 2 Acked 1001 Read 6 Sent Done Done false
104262024-02-04T14:04:53.110Z 3 Acked 1002 Write 1 Sent Done Done false
104272024-02-04T14:04:53.110Z 4 Acked 1003 Write 8 Sent Done Done false
104282024-02-04T14:04:53.110Z 5 Acked 1004 Flush 0 Sent Done Done false
104292024-02-04T14:04:53.110Z 6 Acked 1005 Write 10 Sent Done Done false
104302024-02-04T14:04:53.110Z 7 Acked 1006 Read 9 Sent Done Done false
104312024-02-04T14:04:53.110Z 8 Acked 1007 Flush 0 Sent Done Done false
104322024-02-04T14:04:53.110Z 9 Acked 1008 Write 8 Sent Done Done false
104332024-02-04T14:04:53.110Z 10 Acked 1009 Read 5 Sent Done Done false
104342024-02-04T14:04:53.110Z 11 Acked 1010 Read 7 Sent Done Done false
104352024-02-04T14:04:53.110Z 12 Acked 1011 Write 10 Sent Done Done false
104362024-02-04T14:04:53.110Z 13 Acked 1012 Read 8 Sent Done Done false
104372024-02-04T14:04:53.110Z 14 Acked 1013 Read 7 Sent Done Done false
104382024-02-04T14:04:53.110Z 15 Acked 1014 Write 3 Sent Done Done false
104392024-02-04T14:04:53.111Z 16 Acked 1015 Write 8 Sent Done Done false
104402024-02-04T14:04:53.111Z 17 Acked 1016 Write 10 Sent Done Done false
104412024-02-04T14:04:53.111Z 18 Acked 1017 Read 10 Sent Done Done false
104422024-02-04T14:04:53.111Z 19 Acked 1018 Read 6 Sent Done Done false
104432024-02-04T14:04:53.111Z 20 Acked 1019 Read 8 Sent Done Done false
104442024-02-04T14:04:53.111Z 21 Acked 1020 Read 9 Sent Done Done false
104452024-02-04T14:04:53.111Z 22 Acked 1021 Read 4 Sent Done Done false
104462024-02-04T14:04:53.111Z 23 Acked 1022 Write 3 Sent Done Done false
104472024-02-04T14:04:53.111Z 24 Acked 1023 Write 7 Sent Done Done false
104482024-02-04T14:04:53.111Z 25 Acked 1024 Read 7 Sent Done Done false
104492024-02-04T14:04:53.111Z 26 Acked 1025 Write 2 Sent Done Done false
104502024-02-04T14:04:53.111Z 27 Acked 1026 Write 4 Sent Done Done false
104512024-02-04T14:04:53.111Z 28 Acked 1027 Read 8 Sent Done Done false
104522024-02-04T14:04:53.111Z 29 Acked 1028 Write 2 Sent Done Done false
104532024-02-04T14:04:53.111Z 30 Acked 1029 Read 5 Sent Done Done false
104542024-02-04T14:04:53.111Z STATES DS:0 DS:1 DS:2 TOTAL
104552024-02-04T14:04:53.112Z New 0 0 0 0
104562024-02-04T14:04:53.112Z Sent 30 0 0 30
104572024-02-04T14:04:53.112Z Done 0 30 30 60
104582024-02-04T14:04:53.112Z Skipped 0 0 0 0
104592024-02-04T14:04:53.112Z Error 0 0 0 0
104602024-02-04T14:04:53.112ZLast Flush: 0 1007 1007
104612024-02-04T14:04:53.112ZDownstairs last five completed:
104622024-02-04T14:04:53.112ZUpstairs last five completed: 30 29 28 27 26
104632024-02-04T14:04:53.112Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:52.141255526Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1513}
104642024-02-04T14:04:53.112ZWrote out file "/tmp/test_repair_verify.data"
104652024-02-04T14:04:53.115Z
104662024-02-04T14:04:53.119ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104672024-02-04T14:04:53.147ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104682024-02-04T14:04:53.147Z 0 000-019 87 87 87 139 139 139 F F F
104692024-02-04T14:04:53.147Z 1 020-039 91 91 91 148 148 148 F F F
104702024-02-04T14:04:53.147Z 2 040-059 88 96 96 142 156 156 F F F <---
104712024-02-04T14:04:53.147Z 3 060-079 82 82 82 129 129 129 F T T <---
104722024-02-04T14:04:53.154Z 4 080-099 95 95 95 155 155 155 F T T <---
104732024-02-04T14:04:53.154Z 5 100-119 92 92 92 150 150 150 F F F
104742024-02-04T14:04:53.154Z 6 120-139 88 88 88 142 142 142 F F F
104752024-02-04T14:04:53.154Z 7 140-159 93 93 93 151 151 151 F F F
104762024-02-04T14:04:53.154Z 8 160-179 95 95 95 155 155 155 F F F
104772024-02-04T14:04:53.154Z 9 180-199 94 94 94 153 153 153 F F F
104782024-02-04T14:04:53.154Z 10 200-219 92 92 92 150 150 150 F T T <---
104792024-02-04T14:04:53.154Z 11 220-239 92 92 92 150 150 150 F T T <---
104802024-02-04T14:04:53.154Z 12 240-259 94 94 94 154 154 154 F F F
104812024-02-04T14:04:53.154Z 13 260-279 93 93 93 151 151 151 F F F
104822024-02-04T14:04:53.154Z 14 280-299 94 94 94 153 153 153 F F F
104832024-02-04T14:04:53.154Z 15 300-319 94 94 94 152 152 152 F T T <---
104842024-02-04T14:04:53.154Z 16 320-339 86 86 86 137 137 137 F F F
104852024-02-04T14:04:53.154Z 17 340-359 88 88 88 142 142 142 F F F
104862024-02-04T14:04:53.154Z 18 360-379 90 90 90 145 145 145 F F F
104872024-02-04T14:04:53.154Z 19 380-399 93 93 93 151 151 151 F T T <---
104882024-02-04T14:04:53.154Z 20 400-419 93 93 93 151 151 151 F F F
104892024-02-04T14:04:53.154Z 21 420-439 86 86 86 138 138 138 F T T <---
104902024-02-04T14:04:53.155Z 22 440-459 94 96 96 153 156 156 F T T <---
104912024-02-04T14:04:53.155Z 23 460-479 85 85 85 135 135 135 F F F
104922024-02-04T14:04:53.155Z 24 480-499 94 94 94 152 152 152 F T T <---
104932024-02-04T14:04:53.155Z 25 500-519 94 96 96 152 157 157 F F F <---
104942024-02-04T14:04:53.155Z 26 520-539 93 96 96 151 156 156 F F F <---
104952024-02-04T14:04:53.155Z 27 540-559 94 94 94 152 152 152 F F F
104962024-02-04T14:04:53.155Z 28 560-579 84 84 84 132 132 132 F T T <---
104972024-02-04T14:04:53.155Z 29 580-599 90 90 90 146 146 146 F F F
104982024-02-04T14:04:53.155ZMax gen: 96, Max flush: 157
104992024-02-04T14:04:53.155ZError: Difference in extent metadata found!
105002024-02-04T14:04:53.158ZA Difference in extent metadata is expected here
105012024-02-04T14:04:53.158ZOn loop 48
105022024-02-04T14:04:53.158Z
105032024-02-04T14:04:53.158ZVerifying data now
105042024-02-04T14:04:58.306ZNo change after verify
105052024-02-04T14:04:58.307ZLoop: 48 Downstairs dump after verify (and repair):
105062024-02-04T14:04:58.329ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105072024-02-04T14:04:58.329Z 0 000-019 87 87 87 139 139 139 F F F
105082024-02-04T14:04:58.329Z 1 020-039 91 91 91 148 148 148 F F F
105092024-02-04T14:04:58.329Z 2 040-059 96 96 96 156 156 156 F F F
105102024-02-04T14:04:58.329Z 3 060-079 97 97 97 158 158 158 F F F
105112024-02-04T14:04:58.333Z 4 080-099 97 97 97 158 158 158 F F F
105122024-02-04T14:04:58.333Z 5 100-119 92 92 92 150 150 150 F F F
105132024-02-04T14:04:58.333Z 6 120-139 88 88 88 142 142 142 F F F
105142024-02-04T14:04:58.333Z 7 140-159 93 93 93 151 151 151 F F F
105152024-02-04T14:04:58.333Z 8 160-179 95 95 95 155 155 155 F F F
105162024-02-04T14:04:58.333Z 9 180-199 94 94 94 153 153 153 F F F
105172024-02-04T14:04:58.333Z 10 200-219 97 97 97 158 158 158 F F F
105182024-02-04T14:04:58.333Z 11 220-239 97 97 97 158 158 158 F F F
105192024-02-04T14:04:58.333Z 12 240-259 94 94 94 154 154 154 F F F
105202024-02-04T14:04:58.333Z 13 260-279 93 93 93 151 151 151 F F F
105212024-02-04T14:04:58.333Z 14 280-299 94 94 94 153 153 153 F F F
105222024-02-04T14:04:58.333Z 15 300-319 97 97 97 158 158 158 F F F
105232024-02-04T14:04:58.333Z 16 320-339 86 86 86 137 137 137 F F F
105242024-02-04T14:04:58.333Z 17 340-359 88 88 88 142 142 142 F F F
105252024-02-04T14:04:58.333Z 18 360-379 90 90 90 145 145 145 F F F
105262024-02-04T14:04:58.333Z 19 380-399 97 97 97 158 158 158 F F F
105272024-02-04T14:04:58.333Z 20 400-419 93 93 93 151 151 151 F F F
105282024-02-04T14:04:58.333Z 21 420-439 97 97 97 158 158 158 F F F
105292024-02-04T14:04:58.333Z 22 440-459 97 97 97 158 158 158 F F F
105302024-02-04T14:04:58.333Z 23 460-479 85 85 85 135 135 135 F F F
105312024-02-04T14:04:58.333Z 24 480-499 97 97 97 158 158 158 F F F
105322024-02-04T14:04:58.333Z 25 500-519 96 96 96 157 157 157 F F F
105332024-02-04T14:04:58.333Z 26 520-539 96 96 96 156 156 156 F F F
105342024-02-04T14:04:58.334Z 27 540-559 94 94 94 152 152 152 F F F
105352024-02-04T14:04:58.334Z 28 560-579 97 97 97 158 158 158 F F F
105362024-02-04T14:04:58.334Z 29 580-599 90 90 90 146 146 146 F F F
105372024-02-04T14:04:58.334ZMax gen: 97, Max flush: 158
105382024-02-04T14:04:58.337Z
105392024-02-04T14:04:58.337ZBegin loop 49
105402024-02-04T14:04:58.337ZDownstairs to restart: 1
105412024-02-04T14:04:58.363Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.39560143Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521}
105422024-02-04T14:04:58.363Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396128889Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521}
105432024-02-04T14:04:58.367Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.39617004Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521}
105442024-02-04T14:04:58.367ZCrucible runtime is spawned
105452024-02-04T14:04:58.367Z{"msg":"Crucible 41732871-fd96-4958-8087-58bf84490490 has session id: 486af2e8-3eb0-4802-af3d-3788760bc1d2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396405624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105462024-02-04T14:04:58.367Z{"msg":"Upstairs opts: Upstairs UUID: 41732871-fd96-4958-8087-58bf84490490, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396488211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105472024-02-04T14:04:58.367Z{"msg":"Crucible stats registered with UUID: 41732871-fd96-4958-8087-58bf84490490","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396548737Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105482024-02-04T14:04:58.367Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396573961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521}
105492024-02-04T14:04:58.367Z{"msg":"41732871-fd96-4958-8087-58bf84490490 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396611939Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105502024-02-04T14:04:58.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396633918Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105512024-02-04T14:04:58.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396696447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105522024-02-04T14:04:58.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396766225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105532024-02-04T14:04:58.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396866899Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105542024-02-04T14:04:58.367Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396905668Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105552024-02-04T14:04:58.367Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396938453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105562024-02-04T14:04:58.367Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.396980107Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"io task","client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105572024-02-04T14:04:58.367Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.397250516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"io task","client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105582024-02-04T14:04:58.367Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.397331935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"io task","client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105592024-02-04T14:04:58.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.397357662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"io task","client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105602024-02-04T14:04:58.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.397380649Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"io task","client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105612024-02-04T14:04:58.367Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.39740768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"io task","client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105622024-02-04T14:04:58.367Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.398380927Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105632024-02-04T14:04:58.367Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.398459626Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105642024-02-04T14:04:58.368Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.398630025Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105652024-02-04T14:04:58.368Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.398663802Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105662024-02-04T14:04:58.368Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.398916002Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105672024-02-04T14:04:58.368Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.399007133Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105682024-02-04T14:04:58.372Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.399675481Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105692024-02-04T14:04:58.372Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.399744554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105702024-02-04T14:04:58.372Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.40011833Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105712024-02-04T14:04:58.372Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.400585066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105722024-02-04T14:04:58.372Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.400649536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105732024-02-04T14:04:58.372Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.400676109Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105742024-02-04T14:04:58.372Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.400716603Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105752024-02-04T14:04:58.372Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.400740643Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105762024-02-04T14:04:58.372Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.400784067Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105772024-02-04T14:04:58.372Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401070706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105782024-02-04T14:04:58.373Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401155138Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105792024-02-04T14:04:58.373Z{"msg":"[0]R flush_numbers[0..12]: [139, 148, 156, 158, 158, 150, 142, 151, 155, 153, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401187113Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105802024-02-04T14:04:58.373Z{"msg":"[0]R generation[0..12]: [87, 91, 96, 97, 97, 92, 88, 93, 95, 94, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401231907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105812024-02-04T14:04:58.373Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401257946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105822024-02-04T14:04:58.373Z{"msg":"[1]R flush_numbers[0..12]: [139, 148, 156, 158, 158, 150, 142, 151, 155, 153, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401299172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105832024-02-04T14:04:58.373Z{"msg":"[1]R generation[0..12]: [87, 91, 96, 97, 97, 92, 88, 93, 95, 94, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401323956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105842024-02-04T14:04:58.373Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
105852024-02-04T14:04:58.373Z,"time":"2024-02-04T14:04:57.401366174Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105862024-02-04T14:04:58.373Z{"msg":"[2]R flush_numbers[0..12]: [139, 148, 156, 158, 158, 150, 142, 151, 155, 153, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401392262Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105872024-02-04T14:04:58.373Z{"msg":"[2]R generation[0..12]: [87, 91, 96, 97, 97, 92, 88, 93, 95, 94, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401415507Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105882024-02-04T14:04:58.373Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401457891Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105892024-02-04T14:04:58.373Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401480941Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105902024-02-04T14:04:58.373Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401530682Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105912024-02-04T14:04:58.373Z{"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401553332Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105922024-02-04T14:04:58.373Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401574986Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105932024-02-04T14:04:58.373Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.40159714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105942024-02-04T14:04:58.373Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401618295Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105952024-02-04T14:04:58.373Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401640558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105962024-02-04T14:04:58.373Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401686859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105972024-02-04T14:04:58.373Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401710265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105982024-02-04T14:04:58.373Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401730612Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
105992024-02-04T14:04:58.373Z{"msg":"41732871-fd96-4958-8087-58bf84490490 is now active with session: 486af2e8-3eb0-4802-af3d-3788760bc1d2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401784514Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
106002024-02-04T14:04:58.373Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401805278Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521,"session_id":"486af2e8-3eb0-4802-af3d-3788760bc1d2"}
106012024-02-04T14:04:58.374Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:04:57.401825667Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521}
106022024-02-04T14:04:58.374ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106032024-02-04T14:04:58.374ZLoading write count information from file "/tmp/test_repair_verify.data"
106042024-02-04T14:04:58.374ZRun Repair workload
106052024-02-04T14:04:58.374Z01/30 Flush
106062024-02-04T14:04:58.374Z02/30 Write block 235 len 28672 data: 5 5 8 8 7 8 8
106072024-02-04T14:04:58.377Z03/30 Write block 169 len 20480 data: 8 10 8 12 11
106082024-02-04T14:04:58.377Z04/30 Read block 227 len 36864
106092024-02-04T14:04:58.377Z05/30 Read block 63 len 20480
106102024-02-04T14:04:58.380Z06/30 Write block 183 len 4096 data: 12
106112024-02-04T14:04:58.380Z07/30 Read block 110 len 32768
106122024-02-04T14:04:58.383Z08/30 Read block 436 len 40960
106132024-02-04T14:04:58.383Z09/30 Write block 208 len 36864 data: 6 4 4 6 7 7 9 8 10
106142024-02-04T14:04:58.386Z10/30 Read block 363 len 4096
106152024-02-04T14:04:58.386Z11/30 Read block 30 len 20480
106162024-02-04T14:04:58.386Z12/30 Read block 188 len 24576
106172024-02-04T14:04:58.389Z13/30 Write block 272 len 16384 data: 5 6 5 5
106182024-02-04T14:04:58.389Z14/30 Read block 314 len 24576
106192024-02-04T14:04:58.392Z15/30 Write block 140 len 24576 data: 7 7 4 5 8 9
106202024-02-04T14:04:58.392Z16/30 Read block 392 len 8192
106212024-02-04T14:04:58.398Z17/30 Write block 187 len 12288 data: 10 9 9
106222024-02-04T14:04:58.398Z18/30 Flush
106232024-02-04T14:04:58.401Z19/30 Write block 162 len 24576 data: 7 9 7 8 9 9
106242024-02-04T14:04:58.401Z20/30 Write block 488 len 16384 data: 8 11 11 12
106252024-02-04T14:04:58.401Z21/30 Write block 560 len 16384 data: 10 9 8 9
106262024-02-04T14:04:58.403Z22/30 Read block 272 len 32768
106272024-02-04T14:04:58.403Z23/30 Read block 354 len 40960
106282024-02-04T14:04:58.406Z24/30 Read block 114 len 12288
106292024-02-04T14:04:58.406Z25/30 Write block 443 len 4096 data: 9
106302024-02-04T14:04:58.409Z26/30 Flush
106312024-02-04T14:04:58.412Z27/30 Write block 344 len 28672 data: 7 8 7 8 7 9 7
106322024-02-04T14:04:58.415Z28/30 Write block 401 len 40960 data: 9 10 8 7 10 9 9 8 9 9
106332024-02-04T14:04:58.415Z29/30 Read block 416 len 4096
106342024-02-04T14:04:58.415Z30/30 Read block 352 len 40960
106352024-02-04T14:04:58.418Z----------------------------------------------------------------
106362024-02-04T14:04:58.418Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:29
106372024-02-04T14:04:58.418ZStates: Active Active Active
106382024-02-04T14:04:58.418ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106392024-02-04T14:04:58.418Z 2 Acked 1001 Write 7 Done Done Done false
106402024-02-04T14:04:58.418Z 3 Acked 1002 Write 5 Done Sent Done false
106412024-02-04T14:04:58.418Z 4 Acked 1003 Read 9 Done Sent Done false
106422024-02-04T14:04:58.418Z 5 Acked 1004 Read 5 Done Sent Done false
106432024-02-04T14:04:58.418Z 6 Acked 1005 Write 1 Done Sent Done false
106442024-02-04T14:04:58.418Z 7 Acked 1006 Read 8 Done Sent Done false
106452024-02-04T14:04:58.419Z 8 Acked 1007 Read 10 Done Sent Done false
106462024-02-04T14:04:58.419Z 9 Acked 1008 Write 9 Done Sent Done false
106472024-02-04T14:04:58.419Z 10 Acked 1009 Read 1 Done Sent Done false
106482024-02-04T14:04:58.419Z 11 Acked 1010 Read 5 Done Sent Done false
106492024-02-04T14:04:58.419Z 12 Acked 1011 Read 6 Done Sent Done false
106502024-02-04T14:04:58.419Z 13 Acked 1012 Write 4 Done Sent Done false
106512024-02-04T14:04:58.419Z 14 Acked 1013 Read 6 Done Sent Done false
106522024-02-04T14:04:58.419Z 15 Acked 1014 Write 6 Done Sent Done false
106532024-02-04T14:04:58.419Z 16 Acked 1015 Read 2 Done Sent Done false
106542024-02-04T14:04:58.419Z 17 Acked 1016 Write 3 Done Sent Done false
106552024-02-04T14:04:58.419Z 18 Acked 1017 Flush 0 Done Sent Done false
106562024-02-04T14:04:58.419Z 19 Acked 1018 Write 6 Done Sent Done false
106572024-02-04T14:04:58.419Z 20 Acked 1019 Write 4 Done Sent Done false
106582024-02-04T14:04:58.419Z 21 Acked 1020 Write 4 Done Sent Done false
106592024-02-04T14:04:58.419Z 22 Acked 1021 Read 8 Done Sent Done false
106602024-02-04T14:04:58.419Z 23 Acked 1022 Read 10 Done Sent Done false
106612024-02-04T14:04:58.419Z 24 Acked 1023 Read 3 Done Sent Done false
106622024-02-04T14:04:58.419Z 25 Acked 1024 Write 1 Done Sent Done false
106632024-02-04T14:04:58.419Z 26 Acked 1025 Flush 0 Done Sent Done false
106642024-02-04T14:04:58.419Z 27 Acked 1026 Write 7 Done Sent Done false
106652024-02-04T14:04:58.419Z 28 Acked 1027 Write 10 Done Sent Done false
106662024-02-04T14:04:58.419Z 29 Acked 1028 Read 1 Done Sent Done false
106672024-02-04T14:04:58.419Z 30 Acked 1029 Read 10 Done Sent Done false
106682024-02-04T14:04:58.419Z STATES DS:0 DS:1 DS:2 TOTAL
106692024-02-04T14:04:58.419Z New 0 0 0 0
106702024-02-04T14:04:58.420Z Sent 0 28 0 28
106712024-02-04T14:04:58.420Z Done 29 1 29 59
106722024-02-04T14:04:58.420Z Skipped 0 0 0 0
106732024-02-04T14:04:58.420Z Error 0 0 0 0
106742024-02-04T14:04:58.420ZLast Flush: 1025 1000 1025
106752024-02-04T14:04:58.420ZDownstairs last five completed: 1000
106762024-02-04T14:04:58.420ZUpstairs last five completed: 30 29 28 27 26
106772024-02-04T14:04:58.420Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:04:57.450440292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1521}
106782024-02-04T14:04:58.420ZWrote out file "/tmp/test_repair_verify.data"
106792024-02-04T14:04:58.423Z
106802024-02-04T14:04:58.427ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106812024-02-04T14:04:58.450ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106822024-02-04T14:04:58.450Z 0 000-019 87 87 87 139 139 139 F F F
106832024-02-04T14:04:58.450Z 1 020-039 91 91 91 148 148 148 F F F
106842024-02-04T14:04:58.451Z 2 040-059 96 96 96 156 156 156 F F F
106852024-02-04T14:04:58.451Z 3 060-079 97 97 97 158 158 158 F F F
106862024-02-04T14:04:58.451Z 4 080-099 97 97 97 158 158 158 F F F
106872024-02-04T14:04:58.451Z 5 100-119 92 92 92 150 150 150 F F F
106882024-02-04T14:04:58.454Z 6 120-139 88 88 88 142 142 142 F F F
106892024-02-04T14:04:58.454Z 7 140-159 98 93 98 160 151 160 F F F <---
106902024-02-04T14:04:58.454Z 8 160-179 98 95 98 161 155 161 F F F <---
106912024-02-04T14:04:58.454Z 9 180-199 98 94 98 160 153 160 F F F <---
106922024-02-04T14:04:58.454Z 10 200-219 98 97 98 160 158 160 F F F <---
106932024-02-04T14:04:58.455Z 11 220-239 98 97 98 160 158 160 F T F <---
106942024-02-04T14:04:58.455Z 12 240-259 98 94 98 160 154 160 F T F <---
106952024-02-04T14:04:58.455Z 13 260-279 98 93 98 160 151 160 F F F <---
106962024-02-04T14:04:58.455Z 14 280-299 94 94 94 153 153 153 F F F
106972024-02-04T14:04:58.455Z 15 300-319 97 97 97 158 158 158 F F F
106982024-02-04T14:04:58.455Z 16 320-339 86 86 86 137 137 137 F F F
106992024-02-04T14:04:58.455Z 17 340-359 88 88 88 142 142 142 T F T <---
107002024-02-04T14:04:58.455Z 18 360-379 90 90 90 145 145 145 F F F
107012024-02-04T14:04:58.455Z 19 380-399 97 97 97 158 158 158 F F F
107022024-02-04T14:04:58.455Z 20 400-419 93 93 93 151 151 151 T F T <---
107032024-02-04T14:04:58.455Z 21 420-439 97 97 97 158 158 158 F F F
107042024-02-04T14:04:58.455Z 22 440-459 98 97 98 161 158 161 F F F <---
107052024-02-04T14:04:58.455Z 23 460-479 85 85 85 135 135 135 F F F
107062024-02-04T14:04:58.455Z 24 480-499 98 97 98 161 158 161 F F F <---
107072024-02-04T14:04:58.455Z 25 500-519 96 96 96 157 157 157 F F F
107082024-02-04T14:04:58.455Z 26 520-539 96 96 96 156 156 156 F F F
107092024-02-04T14:04:58.455Z 27 540-559 94 94 94 152 152 152 F F F
107102024-02-04T14:04:58.455Z 28 560-579 98 97 98 161 158 161 F F F <---
107112024-02-04T14:04:58.455Z 29 580-599 90 90 90 146 146 146 F F F
107122024-02-04T14:04:58.455ZMax gen: 98, Max flush: 161
107132024-02-04T14:04:58.455ZError: Difference in extent metadata found!
107142024-02-04T14:04:58.455ZA Difference in extent metadata is expected here
107152024-02-04T14:04:58.455ZOn loop 49
107162024-02-04T14:04:58.456Z
107172024-02-04T14:04:58.458ZVerifying data now
107182024-02-04T14:05:04.519ZNo change after verify
107192024-02-04T14:05:04.519ZLoop: 49 Downstairs dump after verify (and repair):
107202024-02-04T14:05:04.542ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107212024-02-04T14:05:04.542Z 0 000-019 87 87 87 139 139 139 F F F
107222024-02-04T14:05:04.542Z 1 020-039 91 91 91 148 148 148 F F F
107232024-02-04T14:05:04.542Z 2 040-059 96 96 96 156 156 156 F F F
107242024-02-04T14:05:04.542Z 3 060-079 97 97 97 158 158 158 F F F
107252024-02-04T14:05:04.542Z 4 080-099 97 97 97 158 158 158 F F F
107262024-02-04T14:05:04.542Z 5 100-119 92 92 92 150 150 150 F F F
107272024-02-04T14:05:04.546Z 6 120-139 88 88 88 142 142 142 F F F
107282024-02-04T14:05:04.546Z 7 140-159 98 98 98 160 160 160 F F F
107292024-02-04T14:05:04.546Z 8 160-179 98 98 98 161 161 161 F F F
107302024-02-04T14:05:04.546Z 9 180-199 98 98 98 160 160 160 F F F
107312024-02-04T14:05:04.546Z 10 200-219 98 98 98 160 160 160 F F F
107322024-02-04T14:05:04.546Z 11 220-239 98 98 98 160 160 160 F F F
107332024-02-04T14:05:04.546Z 12 240-259 98 98 98 160 160 160 F F F
107342024-02-04T14:05:04.546Z 13 260-279 98 98 98 160 160 160 F F F
107352024-02-04T14:05:04.546Z 14 280-299 94 94 94 153 153 153 F F F
107362024-02-04T14:05:04.547Z 15 300-319 97 97 97 158 158 158 F F F
107372024-02-04T14:05:04.547Z 16 320-339 86 86 86 137 137 137 F F F
107382024-02-04T14:05:04.547Z 17 340-359 99 99 99 162 162 162 F F F
107392024-02-04T14:05:04.547Z 18 360-379 90 90 90 145 145 145 F F F
107402024-02-04T14:05:04.547Z 19 380-399 97 97 97 158 158 158 F F F
107412024-02-04T14:05:04.547Z 20 400-419 99 99 99 162 162 162 F F F
107422024-02-04T14:05:04.547Z 21 420-439 97 97 97 158 158 158 F F F
107432024-02-04T14:05:04.547Z 22 440-459 98 98 98 161 161 161 F F F
107442024-02-04T14:05:04.547Z 23 460-479 85 85 85 135 135 135 F F F
107452024-02-04T14:05:04.547Z 24 480-499 98 98 98 161 161 161 F F F
107462024-02-04T14:05:04.547Z 25 500-519 96 96 96 157 157 157 F F F
107472024-02-04T14:05:04.547Z 26 520-539 96 96 96 156 156 156 F F F
107482024-02-04T14:05:04.547Z 27 540-559 94 94 94 152 152 152 F F F
107492024-02-04T14:05:04.547Z 28 560-579 98 98 98 161 161 161 F F F
107502024-02-04T14:05:04.547Z 29 580-599 90 90 90 146 146 146 F F F
107512024-02-04T14:05:04.547ZMax gen: 99, Max flush: 162
107522024-02-04T14:05:04.547Z
107532024-02-04T14:05:04.547ZBegin loop 50
107542024-02-04T14:05:04.547ZDownstairs to restart: 1
107552024-02-04T14:05:04.576Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.609014373Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529}
107562024-02-04T14:05:04.582Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.609478538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529}
107572024-02-04T14:05:04.582Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.609580381Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529}
107582024-02-04T14:05:04.582Z{"msg":"Crucible 7658e843-47a4-4a7e-85cb-474e8fe7d6b8 has session id: 6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.609621981Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107592024-02-04T14:05:04.582Z{"msg":"Upstairs opts: Upstairs UUID: 7658e843-47a4-4a7e-85cb-474e8fe7d6b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.609674381Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107602024-02-04T14:05:04.582Z{"msg":"Crucible stats registered with UUID: 7658e843-47a4-4a7e-85cb-474e8fe7d6b8","vCrucible runtime is spawned
107612024-02-04T14:05:04.582Z":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.609816699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107622024-02-04T14:05:04.582Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.609885761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529}
107632024-02-04T14:05:04.582Z{"msg":"7658e843-47a4-4a7e-85cb-474e8fe7d6b8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.609973203Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107642024-02-04T14:05:04.582Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610048163Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107652024-02-04T14:05:04.583Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610076355Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107662024-02-04T14:05:04.583Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610144233Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107672024-02-04T14:05:04.583Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610173561Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107682024-02-04T14:05:04.583Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610237364Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107692024-02-04T14:05:04.583Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610266496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107702024-02-04T14:05:04.583Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610289421Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"io task","client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107712024-02-04T14:05:04.583Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610313421Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"io task","client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107722024-02-04T14:05:04.583Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610336737Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"io task","client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107732024-02-04T14:05:04.583Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610509536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"io task","client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107742024-02-04T14:05:04.583Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.61062857Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"io task","client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107752024-02-04T14:05:04.583Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.610704687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"io task","client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107762024-02-04T14:05:04.583Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.611738468Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107772024-02-04T14:05:04.583Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.611867796Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107782024-02-04T14:05:04.583Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.612021266Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107792024-02-04T14:05:04.583Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.612094982Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107802024-02-04T14:05:04.583Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.612559978Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107812024-02-04T14:05:04.583Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.612811819Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107822024-02-04T14:05:04.583Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.612921195Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107832024-02-04T14:05:04.583Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.612966742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107842024-02-04T14:05:04.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.61345472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107852024-02-04T14:05:04.588Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.613535513Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107862024-02-04T14:05:04.588Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.613704226Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107872024-02-04T14:05:04.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.61399257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107882024-02-04T14:05:04.588Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.614062473Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107892024-02-04T14:05:04.588Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.61413497Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107902024-02-04T14:05:04.588Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.614388553Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107912024-02-04T14:05:04.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615073048Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107922024-02-04T14:05:04.588Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615153478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107932024-02-04T14:05:04.588Z{"msg":"[0]R flush_numbers[0..12]: [139, 148, 156, 158, 158, 150, 142, 160, 161, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615181925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107942024-02-04T14:05:04.589Z{"msg":"[0]R generation[0..12]: [87, 91, 96, 97, 97, 92, 88, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615242297Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107952024-02-04T14:05:04.589Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.61530458Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107962024-02-04T14:05:04.589Z{"msg":"[1]R flush_numbers[0..12]: [139, 148, 156, 158, 158, 150, 142, 160, 161, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615338825Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
107972024-02-04T14:05:04.589Zsession_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107982024-02-04T14:05:04.589Z{"msg":"[1]R generation[0..12]: [87, 91, 96, 97, 97, 92, 88, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615446197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
107992024-02-04T14:05:04.589Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615508371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108002024-02-04T14:05:04.589Z{"msg":"[2]R flush_numbers[0..12]: [139, 148, 156, 158, 158, 150, 142, 160, 161, 160, 160, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615535021Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108012024-02-04T14:05:04.589Z{"msg":"[2]R generation[0..12]: [87, 91, 96, 97, 97, 92, 88, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615600534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108022024-02-04T14:05:04.589Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615629267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108032024-02-04T14:05:04.589Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615657651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108042024-02-04T14:05:04.589Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615731967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108052024-02-04T14:05:04.589Z{"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615757084Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108062024-02-04T14:05:04.589Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615815087Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108072024-02-04T14:05:04.589Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615842169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108082024-02-04T14:05:04.589Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.61589785Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108092024-02-04T14:05:04.589Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615927554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108102024-02-04T14:05:04.589Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.615993248Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108112024-02-04T14:05:04.589Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.616028377Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108122024-02-04T14:05:04.589Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.616081241Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108132024-02-04T14:05:04.589Z{"msg":"7658e843-47a4-4a7e-85cb-474e8fe7d6b8 is now active with session: 6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.616106063Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108142024-02-04T14:05:04.589Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.616168995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529,"session_id":"6ad5f6ce-ac21-4fec-8b94-2d90220eb4e1"}
108152024-02-04T14:05:04.589Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:03.616195397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529}
108162024-02-04T14:05:04.589ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108172024-02-04T14:05:04.590ZLoading write count information from file "/tmp/test_repair_verify.data"
108182024-02-04T14:05:04.590ZRun Repair workload
108192024-02-04T14:05:04.590Z01/30 Flush
108202024-02-04T14:05:04.590Z02/30 Write block 295 len 28672 data: 6 5 5 6 6 4 4
108212024-02-04T14:05:04.593Z03/30 Write block 56 len 40960 data: 6 7 5 7 9 8 8 6 6 5
108222024-02-04T14:05:04.593Z04/30 Write block 568 len 24576 data: 7 7 6 6 6 6
108232024-02-04T14:05:04.593Z05/30 Write block 197 len 16384 data: 7 9 10 11
108242024-02-04T14:05:04.593Z06/30 Write block 134 len 24576 data: 5 5 3 3 3 3
108252024-02-04T14:05:04.593Z07/30 Write block 139 len 8192 data: 4 8
108262024-02-04T14:05:04.596Z08/30 Read block 549 len 12288
108272024-02-04T14:05:04.596Z09/30 Read block 440 len 16384
108282024-02-04T14:05:04.601Z10/30 Write block 63 len 40960 data: 7 7 6 4 6 7 5 5 7 7
108292024-02-04T14:05:04.601Z11/30 Flush
108302024-02-04T14:05:04.604Z12/30 Write block 177 len 20480 data: 10 8 11 10 11
108312024-02-04T14:05:04.607Z13/30 Write block 570 len 4096 data: 7
108322024-02-04T14:05:04.607Z14/30 Read block 292 len 28672
108332024-02-04T14:05:04.610Z15/30 Read block 6 len 40960
108342024-02-04T14:05:04.610Z16/30 Write block 559 len 40960 data: 9 11 10 9 10 10 11 11 9 8
108352024-02-04T14:05:04.613Z17/30 Read block 427 len 32768
108362024-02-04T14:05:04.616Z18/30 Read block 192 len 4096
108372024-02-04T14:05:04.616Z19/30 Read block 84 len 12288
108382024-02-04T14:05:04.616Z20/30 Flush
108392024-02-04T14:05:04.619Z21/30 Flush
108402024-02-04T14:05:04.623Z22/30 Write block 403 len 20480 data: 9 8 11 10 10
108412024-02-04T14:05:04.623Z23/30 Read block 319 len 32768
108422024-02-04T14:05:04.623Z24/30 Write block 246 len 12288 data: 6 7 11
108432024-02-04T14:05:04.627Z25/30 Write block 597 len 4096 data: 4
108442024-02-04T14:05:04.627Z26/30 Flush
108452024-02-04T14:05:04.627Z27/30 Write block 523 len 32768 data: 6 7 6 8 7 8 6 5
108462024-02-04T14:05:04.630Z28/30 Read block 292 len 12288
108472024-02-04T14:05:04.630Z29/30 Read block 342 len 4096
108482024-02-04T14:05:04.633Z30/30 Write block 181 len 16384 data: 12 10 13 11
108492024-02-04T14:05:04.634Z----------------------------------------------------------------
108502024-02-04T14:05:04.634Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:29
108512024-02-04T14:05:04.634ZStates: Active Active Active
108522024-02-04T14:05:04.634ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108532024-02-04T14:05:04.634Z 2 Acked 1001 Write 7 Done Done Done false
108542024-02-04T14:05:04.634Z 3 Acked 1002 Write 10 Done Sent Done false
108552024-02-04T14:05:04.634Z 4 Acked 1003 Write 6 Done Sent Done false
108562024-02-04T14:05:04.634Z 5 Acked 1004 Write 4 Done Sent Done false
108572024-02-04T14:05:04.634Z 6 Acked 1005 Write 6 Done Sent Done false
108582024-02-04T14:05:04.634Z 7 Acked 1006 Write 2 Done Sent Done false
108592024-02-04T14:05:04.634Z 8 Acked 1007 Read 3 Done Sent Done false
108602024-02-04T14:05:04.634Z 9 Acked 1008 Read 4 Done Sent Done false
108612024-02-04T14:05:04.634Z 10 Acked 1009 Write 10 Done Sent Done false
108622024-02-04T14:05:04.634Z 11 Acked 1010 Flush 0 Done Sent Done false
108632024-02-04T14:05:04.634Z 12 Acked 1011 Write 5 Done Sent Done false
108642024-02-04T14:05:04.634Z 13 Acked 1012 Write 1 Done Sent Done false
108652024-02-04T14:05:04.634Z 14 Acked 1013 Read 7 Done Sent Done false
108662024-02-04T14:05:04.634Z 15 Acked 1014 Read 10 Done Sent Done false
108672024-02-04T14:05:04.634Z 16 Acked 1015 Write 10 Done Sent Done false
108682024-02-04T14:05:04.634Z 17 Acked 1016 Read 8 Done Sent Done false
108692024-02-04T14:05:04.634Z 18 Acked 1017 Read 1 Done Sent Done false
108702024-02-04T14:05:04.634Z 19 Acked 1018 Read 3 Done Sent Done false
108712024-02-04T14:05:04.634Z 20 Acked 1019 Flush 0 Done Sent Done false
108722024-02-04T14:05:04.634Z 21 Acked 1020 Flush 0 Done Sent Done false
108732024-02-04T14:05:04.635Z 22 Acked 1021 Write 5 Done Sent Done false
108742024-02-04T14:05:04.635Z 23 Acked 1022 Read 8 Done Sent Done false
108752024-02-04T14:05:04.635Z 24 Acked 1023 Write 3 Done Sent Done false
108762024-02-04T14:05:04.635Z 25 Acked 1024 Write 1 Done Sent Done false
108772024-02-04T14:05:04.635Z 26 Acked 1025 Flush 0 Done Sent Done false
108782024-02-04T14:05:04.635Z 27 Acked 1026 Write 8 Done Sent Done false
108792024-02-04T14:05:04.635Z 28 Acked 1027 Read 3 Done Sent Done false
108802024-02-04T14:05:04.635Z 29 Acked 1028 Read 1 Done Sent Done false
108812024-02-04T14:05:04.635Z 30 Acked 1029 Write 4 Sent Sent Sent false
108822024-02-04T14:05:04.635Z STATES DS:0 DS:1 DS:2 TOTAL
108832024-02-04T14:05:04.635Z New 0 0 0 0
108842024-02-04T14:05:04.635Z Sent 1 28 1 30
108852024-02-04T14:05:04.635Z Done 28 1 28 57
108862024-02-04T14:05:04.635Z Skipped 0 0 0 0
108872024-02-04T14:05:04.635Z Error 0 0 0 0
108882024-02-04T14:05:04.635ZLast Flush: 1025 1000 1025
108892024-02-04T14:05:04.635ZDownstairs last five completed: 1000
108902024-02-04T14:05:04.635ZUpstairs last five completed: 30 29 28 27 26
108912024-02-04T14:05:04.635Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:03.665151285Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1529}
108922024-02-04T14:05:04.635ZWrote out file "/tmp/test_repair_verify.data"
108932024-02-04T14:05:04.639Z
108942024-02-04T14:05:04.647ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108952024-02-04T14:05:04.670ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108962024-02-04T14:05:04.670Z 0 000-019 87 87 87 139 139 139 F F F
108972024-02-04T14:05:04.670Z 1 020-039 91 91 91 148 148 148 F F F
108982024-02-04T14:05:04.671Z 2 040-059 100 96 100 164 156 164 F F F <---
108992024-02-04T14:05:04.671Z 3 060-079 100 97 100 164 158 164 F F F <---
109002024-02-04T14:05:04.671Z 4 080-099 97 97 97 158 158 158 F F F
109012024-02-04T14:05:04.674Z 5 100-119 92 92 92 150 150 150 F F F
109022024-02-04T14:05:04.674Z 6 120-139 100 88 100 164 142 164 F F F <---
109032024-02-04T14:05:04.674Z 7 140-159 100 98 100 164 160 164 F F F <---
109042024-02-04T14:05:04.674Z 8 160-179 100 98 100 165 161 165 F F F <---
109052024-02-04T14:05:04.674Z 9 180-199 100 98 100 165 160 165 T F T <---
109062024-02-04T14:05:04.674Z 10 200-219 100 98 100 164 160 164 F F F <---
109072024-02-04T14:05:04.674Z 11 220-239 98 98 98 160 160 160 F F F
109082024-02-04T14:05:04.674Z 12 240-259 100 98 100 167 160 167 F F F <---
109092024-02-04T14:05:04.674Z 13 260-279 98 98 98 160 160 160 F F F
109102024-02-04T14:05:04.674Z 14 280-299 100 94 100 164 153 164 F T F <---
109112024-02-04T14:05:04.675Z 15 300-319 100 97 100 164 158 164 F T F <---
109122024-02-04T14:05:04.675Z 16 320-339 86 86 86 137 137 137 F F F
109132024-02-04T14:05:04.675Z 17 340-359 99 99 99 162 162 162 F F F
109142024-02-04T14:05:04.675Z 18 360-379 90 90 90 145 145 145 F F F
109152024-02-04T14:05:04.675Z 19 380-399 97 97 97 158 158 158 F F F
109162024-02-04T14:05:04.675Z 20 400-419 100 99 100 167 162 167 F F F <---
109172024-02-04T14:05:04.675Z 21 420-439 97 97 97 158 158 158 F F F
109182024-02-04T14:05:04.675Z 22 440-459 98 98 98 161 161 161 F F F
109192024-02-04T14:05:04.675Z 23 460-479 85 85 85 135 135 135 F F F
109202024-02-04T14:05:04.675Z 24 480-499 98 98 98 161 161 161 F F F
109212024-02-04T14:05:04.675Z 25 500-519 96 96 96 157 157 157 F F F
109222024-02-04T14:05:04.675Z 26 520-539 96 96 96 156 156 156 T F T <---
109232024-02-04T14:05:04.675Z 27 540-559 100 94 100 165 152 165 F F F <---
109242024-02-04T14:05:04.675Z 28 560-579 100 98 100 165 161 165 F F F <---
109252024-02-04T14:05:04.675Z 29 580-599 100 90 100 167 146 167 F F F <---
109262024-02-04T14:05:04.675ZMax gen: 100, Max flush: 167
109272024-02-04T14:05:04.675ZError: Difference in extent metadata found!
109282024-02-04T14:05:04.679ZA Difference in extent metadata is expected here
109292024-02-04T14:05:04.679ZOn loop 50
109302024-02-04T14:05:04.679Z
109312024-02-04T14:05:04.679ZVerifying data now
109322024-02-04T14:05:10.937ZNo change after verify
109332024-02-04T14:05:10.937ZLoop: 50 Downstairs dump after verify (and repair):
109342024-02-04T14:05:10.972ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109352024-02-04T14:05:10.972Z 0 000-019 87 87 87 139 139 139 F F F
109362024-02-04T14:05:10.972Z 1 020-039 91 91 91 148 148 148 F F F
109372024-02-04T14:05:10.972Z 2 040-059 100 100 100 164 164 164 F F F
109382024-02-04T14:05:10.972Z 3 060-079 100 100 100 164 164 164 F F F
109392024-02-04T14:05:10.972Z 4 080-099 97 97 97 158 158 158 F F F
109402024-02-04T14:05:10.973Z 5 100-119 92 92 92 150 150 150 F F F
109412024-02-04T14:05:10.977Z 6 120-139 100 100 100 164 164 164 F F F
109422024-02-04T14:05:10.977Z 7 140-159 100 100 100 164 164 164 F F F
109432024-02-04T14:05:10.977Z 8 160-179 100 100 100 165 165 165 F F F
109442024-02-04T14:05:10.977Z 9 180-199 101 101 101 168 168 168 F F F
109452024-02-04T14:05:10.977Z 10 200-219 100 100 100 164 164 164 F F F
109462024-02-04T14:05:10.977Z 11 220-239 98 98 98 160 160 160 F F F
109472024-02-04T14:05:10.977Z 12 240-259 100 100 100 167 167 167 F F F
109482024-02-04T14:05:10.977Z 13 260-279 98 98 98 160 160 160 F F F
109492024-02-04T14:05:10.977Z 14 280-299 100 100 100 164 164 164 F F F
109502024-02-04T14:05:10.977Z 15 300-319 100 100 100 164 164 164 F F F
109512024-02-04T14:05:10.977Z 16 320-339 86 86 86 137 137 137 F F F
109522024-02-04T14:05:10.978Z 17 340-359 99 99 99 162 162 162 F F F
109532024-02-04T14:05:10.978Z 18 360-379 90 90 90 145 145 145 F F F
109542024-02-04T14:05:10.978Z 19 380-399 97 97 97 158 158 158 F F F
109552024-02-04T14:05:10.978Z 20 400-419 100 100 100 167 167 167 F F F
109562024-02-04T14:05:10.978Z 21 420-439 97 97 97 158 158 158 F F F
109572024-02-04T14:05:10.978Z 22 440-459 98 98 98 161 161 161 F F F
109582024-02-04T14:05:10.978Z 23 460-479 85 85 85 135 135 135 F F F
109592024-02-04T14:05:10.978Z 24 480-499 98 98 98 161 161 161 F F F
109602024-02-04T14:05:10.978Z 25 500-519 96 96 96 157 157 157 F F F
109612024-02-04T14:05:10.978Z 26 520-539 101 101 101 168 168 168 F F F
109622024-02-04T14:05:10.978Z 27 540-559 100 100 100 165 165 165 F F F
109632024-02-04T14:05:10.978Z 28 560-579 100 100 100 165 165 165 F F F
109642024-02-04T14:05:10.978Z 29 580-599 100 100 100 167 167 167 F F F
109652024-02-04T14:05:10.979ZMax gen: 101, Max flush: 168
109662024-02-04T14:05:10.979Z
109672024-02-04T14:05:10.979ZBegin loop 51
109682024-02-04T14:05:10.979ZDownstairs to restart: 2
109692024-02-04T14:05:10.997Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.029642777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537}
109702024-02-04T14:05:11.006Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030204763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537}
109712024-02-04T14:05:11.006Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030287862Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537}
109722024-02-04T14:05:11.006Z{"msg":"Crucible bc0e17b3-79ef-43eb-a57e-f62aa0aa8a0e has session id: 0e68940d-42bd-48d6-b6c8-8611662f11b0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030358438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,Crucible runtime is spawned
109732024-02-04T14:05:11.006Z"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109742024-02-04T14:05:11.006Z{"msg":"Upstairs opts: Upstairs UUID: bc0e17b3-79ef-43eb-a57e-f62aa0aa8a0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030496298Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109752024-02-04T14:05:11.006Z{"msg":"Crucible stats registered with UUID: bc0e17b3-79ef-43eb-a57e-f62aa0aa8a0e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030566491Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109762024-02-04T14:05:11.006Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.03059171Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537}
109772024-02-04T14:05:11.006Z{"msg":"bc0e17b3-79ef-43eb-a57e-f62aa0aa8a0e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030696185Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109782024-02-04T14:05:11.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030768401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109792024-02-04T14:05:11.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030839984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109802024-02-04T14:05:11.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030866972Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109812024-02-04T14:05:11.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030892013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109822024-02-04T14:05:11.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.030968117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109832024-02-04T14:05:11.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.03101026Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109842024-02-04T14:05:11.007Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.0310494Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109852024-02-04T14:05:11.007Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.031256096Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109862024-02-04T14:05:11.007Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.031336716Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109872024-02-04T14:05:11.007Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.03147545Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109882024-02-04T14:05:11.007Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.031544563Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109892024-02-04T14:05:11.007Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.031573658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109902024-02-04T14:05:11.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.032396884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109912024-02-04T14:05:11.008Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.032436921Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109922024-02-04T14:05:11.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.032583216Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109932024-02-04T14:05:11.008Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.032731324Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109942024-02-04T14:05:11.008Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.033244131Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109952024-02-04T14:05:11.008Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.033898585Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109962024-02-04T14:05:11.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.034331764Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109972024-02-04T14:05:11.008Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.034457228Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109982024-02-04T14:05:11.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.034565244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
109992024-02-04T14:05:11.008Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.034613724Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110002024-02-04T14:05:11.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.034639453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110012024-02-04T14:05:11.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.034703246Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110022024-02-04T14:05:11.009Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.034749457Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110032024-02-04T14:05:11.009Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.034804886Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110042024-02-04T14:05:11.009Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.035218715Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110052024-02-04T14:05:11.009Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.035904809Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110062024-02-04T14:05:11.009Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036004627Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110072024-02-04T14:05:11.009Z{"msg":"[0]R flush_numbers[0..12]: [139, 148, 164, 164, 158, 150, 164, 164, 165, 168, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036083106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110082024-02-04T14:05:11.009Z{"msg":"[0]R generation[0..12]: [87, 91, 100, 100, 97, 92, 100, 100, 100, 101, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036146356Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110092024-02-04T14:05:11.009Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036181381Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110102024-02-04T14:05:11.009Z{"msg":"[1]R flush_numbers[0..12]: [139, 148, 164, 164, 158, 150, 164, 164, 165, 168, 164, 160]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
110112024-02-04T14:05:11.009Z,"time":"2024-02-04T14:05:10.036257646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110122024-02-04T14:05:11.009Z{"msg":"[1]R generation[0..12]: [87, 91, 100, 100, 97, 92, 100, 100, 100, 101, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036324309Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110132024-02-04T14:05:11.009Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036358538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110142024-02-04T14:05:11.009Z{"msg":"[2]R flush_numbers[0..12]: [139, 148, 164, 164, 158, 150, 164, 164, 165, 168, 164, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036429867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110152024-02-04T14:05:11.010Z{"msg":"[2]R generation[0..12]: [87, 91, 100, 100, 97, 92, 100, 100, 100, 101, 100, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036456616Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110162024-02-04T14:05:11.010Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036514773Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110172024-02-04T14:05:11.010Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036544403Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110182024-02-04T14:05:11.010Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036606295Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110192024-02-04T14:05:11.010Z{"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036632214Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110202024-02-04T14:05:11.010Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.03669764Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110212024-02-04T14:05:11.010Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036725044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110222024-02-04T14:05:11.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036755083Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110232024-02-04T14:05:11.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036808702Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110242024-02-04T14:05:11.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110252024-02-04T14:05:11.010Z,"time":"2024-02-04T14:05:10.03683445Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110262024-02-04T14:05:11.010Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036918122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110272024-02-04T14:05:11.011Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.036945055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110282024-02-04T14:05:11.011Z{"msg":"bc0e17b3-79ef-43eb-a57e-f62aa0aa8a0e is now active with session: 0e68940d-42bd-48d6-b6c8-8611662f11b0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.03700231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110292024-02-04T14:05:11.011Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.037027669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537,"session_id":"0e68940d-42bd-48d6-b6c8-8611662f11b0"}
110302024-02-04T14:05:11.011Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:10.037048743Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537}
110312024-02-04T14:05:11.015ZLoading write count information from file "/tmp/test_repair_verify.data"
110322024-02-04T14:05:11.015ZRun Repair workload
110332024-02-04T14:05:11.015Z01/30 Write block 259 len 8192 data: 8 6
110342024-02-04T14:05:11.015Z02/30 Read block 313 len 24576
110352024-02-04T14:05:11.015Z03/30 Read block 71 len 8192
110362024-02-04T14:05:11.015Z04/30 Flush
110372024-02-04T14:05:11.015Z05/30 Write block 434 len 16384 data: 10 10 9 7
110382024-02-04T14:05:11.015Z06/30 Read block 132 len 8192
110392024-02-04T14:05:11.019Z07/30 Read block 516 len 20480
110402024-02-04T14:05:11.019Z08/30 Write block 200 len 36864 data: 12 12 12 12 13 10 9 8 7
110412024-02-04T14:05:11.019Z09/30 Read block 428 len 32768
110422024-02-04T14:05:11.022Z10/30 Flush
110432024-02-04T14:05:11.025Z11/30 Write block 246 len 12288 data: 7 8 12
110442024-02-04T14:05:11.025Z12/30 Read block 314 len 32768
110452024-02-04T14:05:11.028Z13/30 Write block 36 len 28672 data: 9 8 6 6 5 5 7
110462024-02-04T14:05:11.028Z14/30 Write block 299 len 16384 data: 7 5 5 6
110472024-02-04T14:05:11.028Z15/30 Read block 73 len 28672
110482024-02-04T14:05:11.035Z16/30 Read block 420 len 28672
110492024-02-04T14:05:11.035Z17/30 Read block 406 len 36864
110502024-02-04T14:05:11.035Z18/30 Read block 267 len 20480
110512024-02-04T14:05:11.038Z19/30 Write block 164 len 24576 data: 8 9 10 10 9 9
110522024-02-04T14:05:11.038Z20/30 Read block 33 len 36864
110532024-02-04T14:05:11.038Z21/30 Write block 348 len 16384 data: 8 10 8 9
110542024-02-04T14:05:11.041Z22/30 Write block 32 len 16384 data: 6 7 6 8
110552024-02-04T14:05:11.041Z23/30 Write block 528 len 20480 data: 9 7 6 4 4
110562024-02-04T14:05:11.041Z24/30 Flush
110572024-02-04T14:05:11.048Z25/30 Write block 534 len 4096 data: 7
110582024-02-04T14:05:11.051Z26/30 Write block 430 len 12288 data: 8 7 8
110592024-02-04T14:05:11.051Z27/30 Write block 440 len 32768 data: 7 8 8 10 9 9 10 10
110602024-02-04T14:05:11.051Z28/30 Write block 444 len 32768 data: 10 10 11 11 10 9 9 10
110612024-02-04T14:05:11.051Z29/30 Write block 43 len 20480 data: 7 8 8 8 6
110622024-02-04T14:05:11.055Z30/30 Read block 390 len 24576
110632024-02-04T14:05:11.055Z----------------------------------------------------------------
110642024-02-04T14:05:11.055Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110652024-02-04T14:05:11.055ZStates: Active Active Active
110662024-02-04T14:05:11.055ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110672024-02-04T14:05:11.055Z 1 Acked 1000 Write 2 Done Done Done false
110682024-02-04T14:05:11.055Z 2 Acked 1001 Read 6 Done Done Done false
110692024-02-04T14:05:11.055Z 3 Acked 1002 Read 2 Done Done Sent false
110702024-02-04T14:05:11.055Z 4 Acked 1003 Flush 0 Done Done Sent false
110712024-02-04T14:05:11.055Z 5 Acked 1004 Write 4 Done Done Sent false
110722024-02-04T14:05:11.055Z 6 Acked 1005 Read 2 Done Done Sent false
110732024-02-04T14:05:11.056Z 7 Acked 1006 Read 5 Done Done Sent false
110742024-02-04T14:05:11.056Z 8 Acked 1007 Write 9 Done Done Sent false
110752024-02-04T14:05:11.056Z 9 Acked 1008 Read 8 Done Done Sent false
110762024-02-04T14:05:11.056Z 10 Acked 1009 Flush 0 Done Done Sent false
110772024-02-04T14:05:11.056Z 11 Acked 1010 Write 3 Done Done Sent false
110782024-02-04T14:05:11.056Z 12 Acked 1011 Read 8 Done Done Sent false
110792024-02-04T14:05:11.056Z 13 Acked 1012 Write 7 Done Done Sent false
110802024-02-04T14:05:11.056Z 14 Acked 1013 Write 4 Done Done Sent false
110812024-02-04T14:05:11.056Z 15 Acked 1014 Read 7 Done Done Sent false
110822024-02-04T14:05:11.056Z 16 Acked 1015 Read 7 Done Done Sent false
110832024-02-04T14:05:11.056Z 17 Acked 1016 Read 9 Done Done Sent false
110842024-02-04T14:05:11.056Z 18 Acked 1017 Read 5 Done Done Sent false
110852024-02-04T14:05:11.056Z 19 Acked 1018 Write 6 Done Done Sent false
110862024-02-04T14:05:11.057Z 20 Acked 1019 Read 9 Done Done Sent false
110872024-02-04T14:05:11.057Z 21 Acked 1020 Write 4 Done Done Sent false
110882024-02-04T14:05:11.057Z 22 Acked 1021 Write 4 Done Done Sent false
110892024-02-04T14:05:11.057Z 23 Acked 1022 Write 5 Done Done Sent false
110902024-02-04T14:05:11.057Z 24 Acked 1023 Flush 0 Done Done Sent false
110912024-02-04T14:05:11.057Z 25 Acked 1024 Write 1 Done Done Sent false
110922024-02-04T14:05:11.057Z 26 Acked 1025 Write 3 Done Done Sent false
110932024-02-04T14:05:11.057Z 27 Acked 1026 Write 8 Done Done Sent false
110942024-02-04T14:05:11.057Z 28 Acked 1027 Write 8 Done Done Sent false
110952024-02-04T14:05:11.057Z 29 Acked 1028 Write 5 Done Done Sent false
110962024-02-04T14:05:11.057Z 30 Acked 1029 Read 6 Done Sent Sent false
110972024-02-04T14:05:11.057Z STATES DS:0 DS:1 DS:2 TOTAL
110982024-02-04T14:05:11.058Z New 0 0 0 0
110992024-02-04T14:05:11.058Z Sent 0 1 28 29
111002024-02-04T14:05:11.058Z Done 30 29 2 61
111012024-02-04T14:05:11.058Z Skipped 0 0 0 0
111022024-02-04T14:05:11.058Z Error 0 0 0 0
111032024-02-04T14:05:11.058ZLast Flush: 1023 1023 0
111042024-02-04T14:05:11.058ZDownstairs last five completed:
111052024-02-04T14:05:11.058ZUpstairs last five completed: 30 29 28 27 26
111062024-02-04T14:05:11.058Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:10.086668554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1537}
111072024-02-04T14:05:11.058ZWrote out file "/tmp/test_repair_verify.data"
111082024-02-04T14:05:11.062Z
111092024-02-04T14:05:11.067ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111102024-02-04T14:05:11.092ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111112024-02-04T14:05:11.092Z 0 000-019 87 87 87 139 139 139 F F F
111122024-02-04T14:05:11.093Z 1 020-039 102 102 91 171 171 148 F F F <---
111132024-02-04T14:05:11.093Z 2 040-059 102 102 100 171 171 164 T T F <---
111142024-02-04T14:05:11.093Z 3 060-079 100 100 100 164 164 164 F F F
111152024-02-04T14:05:11.093Z 4 080-099 97 97 97 158 158 158 F F F
111162024-02-04T14:05:11.096Z 5 100-119 92 92 92 150 150 150 F F F
111172024-02-04T14:05:11.096Z 6 120-139 100 100 100 164 164 164 F F F
111182024-02-04T14:05:11.097Z 7 140-159 100 100 100 164 164 164 F F F
111192024-02-04T14:05:11.097Z 8 160-179 102 102 100 171 171 165 F F F <---
111202024-02-04T14:05:11.097Z 9 180-199 101 101 101 168 168 168 F F F
111212024-02-04T14:05:11.097Z 10 200-219 102 102 100 170 170 164 F F F <---
111222024-02-04T14:05:11.097Z 11 220-239 98 98 98 160 160 160 F F F
111232024-02-04T14:05:11.097Z 12 240-259 102 102 100 171 171 167 F F T <---
111242024-02-04T14:05:11.097Z 13 260-279 102 102 98 169 169 160 F F T <---
111252024-02-04T14:05:11.097Z 14 280-299 102 102 100 171 171 164 F F F <---
111262024-02-04T14:05:11.097Z 15 300-319 102 102 100 171 171 164 F F F <---
111272024-02-04T14:05:11.097Z 16 320-339 86 86 86 137 137 137 F F F
111282024-02-04T14:05:11.097Z 17 340-359 102 102 99 171 171 162 F F F <---
111292024-02-04T14:05:11.097Z 18 360-379 90 90 90 145 145 145 F F F
111302024-02-04T14:05:11.098Z 19 380-399 97 97 97 158 158 158 F F F
111312024-02-04T14:05:11.098Z 20 400-419 100 100 100 167 167 167 F F F
111322024-02-04T14:05:11.098Z 21 420-439 102 102 97 170 170 158 T T F <---
111332024-02-04T14:05:11.098Z 22 440-459 98 98 98 161 161 161 T T F <---
111342024-02-04T14:05:11.098Z 23 460-479 85 85 85 135 135 135 F F F
111352024-02-04T14:05:11.098Z 24 480-499 98 98 98 161 161 161 F F F
111362024-02-04T14:05:11.098Z 25 500-519 96 96 96 157 157 157 F F F
111372024-02-04T14:05:11.098Z 26 520-539 102 102 101 171 171 168 T T F <---
111382024-02-04T14:05:11.098Z 27 540-559 100 100 100 165 165 165 F F F
111392024-02-04T14:05:11.098Z 28 560-579 100 100 100 165 165 165 F F F
111402024-02-04T14:05:11.098Z 29 580-599 100 100 100 167 167 167 F F F
111412024-02-04T14:05:11.098ZMax gen: 102, Max flush: 171
111422024-02-04T14:05:11.098ZError: Difference in extent metadata found!
111432024-02-04T14:05:11.102ZA Difference in extent metadata is expected here
111442024-02-04T14:05:11.102ZOn loop 51
111452024-02-04T14:05:11.102Z
111462024-02-04T14:05:11.102ZVerifying data now
111472024-02-04T14:05:13.150ZNo change after verify
111482024-02-04T14:05:13.150ZLoop: 51 Downstairs dump after verify (and repair):
111492024-02-04T14:05:13.173ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111502024-02-04T14:05:13.173Z 0 000-019 87 87 87 139 139 139 F F F
111512024-02-04T14:05:13.173Z 1 020-039 102 102 102 171 171 171 F F F
111522024-02-04T14:05:13.173Z 2 040-059 103 103 103 172 172 172 F F F
111532024-02-04T14:05:13.173Z 3 060-079 100 100 100 164 164 164 F F F
111542024-02-04T14:05:13.173Z 4 080-099 97 97 97 158 158 158 F F F
111552024-02-04T14:05:13.176Z 5 100-119 92 92 92 150 150 150 F F F
111562024-02-04T14:05:13.176Z 6 120-139 100 100 100 164 164 164 F F F
111572024-02-04T14:05:13.176Z 7 140-159 100 100 100 164 164 164 F F F
111582024-02-04T14:05:13.176Z 8 160-179 102 102 102 171 171 171 F F F
111592024-02-04T14:05:13.176Z 9 180-199 101 101 101 168 168 168 F F F
111602024-02-04T14:05:13.176Z 10 200-219 102 102 102 170 170 170 F F F
111612024-02-04T14:05:13.176Z 11 220-239 98 98 98 160 160 160 F F F
111622024-02-04T14:05:13.176Z 12 240-259 102 102 102 171 171 171 F F F
111632024-02-04T14:05:13.177Z 13 260-279 102 102 102 169 169 169 F F F
111642024-02-04T14:05:13.177Z 14 280-299 102 102 102 171 171 171 F F F
111652024-02-04T14:05:13.177Z 15 300-319 102 102 102 171 171 171 F F F
111662024-02-04T14:05:13.177Z 16 320-339 86 86 86 137 137 137 F F F
111672024-02-04T14:05:13.177Z 17 340-359 102 102 102 171 171 171 F F F
111682024-02-04T14:05:13.177Z 18 360-379 90 90 90 145 145 145 F F F
111692024-02-04T14:05:13.177Z 19 380-399 97 97 97 158 158 158 F F F
111702024-02-04T14:05:13.177Z 20 400-419 100 100 100 167 167 167 F F F
111712024-02-04T14:05:13.177Z 21 420-439 103 103 103 172 172 172 F F F
111722024-02-04T14:05:13.177Z 22 440-459 103 103 103 172 172 172 F F F
111732024-02-04T14:05:13.177Z 23 460-479 85 85 85 135 135 135 F F F
111742024-02-04T14:05:13.177Z 24 480-499 98 98 98 161 161 161 F F F
111752024-02-04T14:05:13.177Z 25 500-519 96 96 96 157 157 157 F F F
111762024-02-04T14:05:13.177Z 26 520-539 103 103 103 172 172 172 F F F
111772024-02-04T14:05:13.177Z 27 540-559 100 100 100 165 165 165 F F F
111782024-02-04T14:05:13.177Z 28 560-579 100 100 100 165 165 165 F F F
111792024-02-04T14:05:13.177Z 29 580-599 100 100 100 167 167 167 F F F
111802024-02-04T14:05:13.177ZMax gen: 103, Max flush: 172
111812024-02-04T14:05:13.180Z
111822024-02-04T14:05:13.180ZBegin loop 52
111832024-02-04T14:05:13.180ZDownstairs to restart: 0
111842024-02-04T14:05:13.207Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.239890866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545}
111852024-02-04T14:05:13.207Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240402263Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545}
111862024-02-04T14:05:13.208Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240486868Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545}
111872024-02-04T14:05:13.211Z{"msg":"Crucible 42246591-cf69-445c-ba73-ec1e0e122f31 has session id: fc3ed36a-f391-477d-81ac-8694ad7dded6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240526625Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111882024-02-04T14:05:13.211Z{"msg":"Upstairs opts: Upstairs UUID: 42246591-cf69-445c-ba73-ec1e0e122f31, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240577205Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":Crucible runtime is spawned
111892024-02-04T14:05:13.212Z"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111902024-02-04T14:05:13.212Z{"msg":"Crucible stats registered with UUID: 42246591-cf69-445c-ba73-ec1e0e122f31","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240655995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111912024-02-04T14:05:13.212Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240732521Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545}
111922024-02-04T14:05:13.212Z{"msg":"42246591-cf69-445c-ba73-ec1e0e122f31 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240839438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111932024-02-04T14:05:13.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240892151Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111942024-02-04T14:05:13.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240918226Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111952024-02-04T14:05:13.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240941117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111962024-02-04T14:05:13.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240964524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111972024-02-04T14:05:13.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.240988442Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111982024-02-04T14:05:13.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.241019997Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
111992024-02-04T14:05:13.212Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.241044538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"io task","client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112002024-02-04T14:05:13.212Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.241069964Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"io task","client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112012024-02-04T14:05:13.212Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.241092931Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"io task","client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112022024-02-04T14:05:13.212Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.241437175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"io task","client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112032024-02-04T14:05:13.212Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.241543168Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"io task","client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112042024-02-04T14:05:13.212Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.241625793Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"io task","client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112052024-02-04T14:05:13.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.242479252Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112062024-02-04T14:05:13.212Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.242564022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112072024-02-04T14:05:13.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.24273982Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112082024-02-04T14:05:13.212Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.242864166Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112092024-02-04T14:05:13.212Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.243439731Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112102024-02-04T14:05:13.212Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.243703795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112112024-02-04T14:05:13.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.243839455Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112122024-02-04T14:05:13.213Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.243910207Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112132024-02-04T14:05:13.217Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.244441502Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112142024-02-04T14:05:13.217Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.244568791Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112152024-02-04T14:05:13.217Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.24461185Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112162024-02-04T14:05:13.217Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.244685222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112172024-02-04T14:05:13.217Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.244766998Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112182024-02-04T14:05:13.217Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.244816631Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112192024-02-04T14:05:13.217Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.245096134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112202024-02-04T14:05:13.217Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.245766054Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112212024-02-04T14:05:13.217Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.245824802Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112222024-02-04T14:05:13.217Z{"msg":"[0]R flush_numbers[0..12]: [139, 171, 172, 164, 158, 150, 164, 164, 171, 168, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.245851691Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112232024-02-04T14:05:13.217Z{"msg":"[0]R generation[0..12]: [87, 102, 103, 100, 97, 92, 100, 100, 102, 101, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.245911848Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112242024-02-04T14:05:13.217Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.245947791Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112252024-02-04T14:05:13.217Z{"msg":"[1]R flush_numbers[0..12]: [139, 171, 172, 164, 158, 150, 164, 164, 171, 168, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246006915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112262024-02-04T14:05:13.218Z{"msg":"[1]R generation[0..12]: [87, 102, 103, 100, 97, 92, 100, 100, 102, 101, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246033695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112272024-02-04T14:05:13.218Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246057848Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
112282024-02-04T14:05:13.218Zfc3ed36a-f391-477d-81ac-8694ad7dded6"}
112292024-02-04T14:05:13.218Z{"msg":"[2]R flush_numbers[0..12]: [139, 171, 172, 164, 158, 150, 164, 164, 171, 168, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246085198Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112302024-02-04T14:05:13.218Z{"msg":"[2]R generation[0..12]: [87, 102, 103, 100, 97, 92, 100, 100, 102, 101, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246145946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112312024-02-04T14:05:13.218Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246175196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112322024-02-04T14:05:13.218Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.24623473Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112332024-02-04T14:05:13.218Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246261666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112342024-02-04T14:05:13.218Z{"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246322504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112352024-02-04T14:05:13.218Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246353916Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112362024-02-04T14:05:13.218Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246410869Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112372024-02-04T14:05:13.218Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246435513Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112382024-02-04T14:05:13.218Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246501198Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112392024-02-04T14:05:13.218Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246532546Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112402024-02-04T14:05:13.218Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246595516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112412024-02-04T14:05:13.218Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246620072Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112422024-02-04T14:05:13.218Z{"msg":"42246591-cf69-445c-ba73-ec1e0e122f31 is now active with session: fc3ed36a-f391-477d-81ac-8694ad7dded6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246677698Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112432024-02-04T14:05:13.218Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246706874Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545,"session_id":"fc3ed36a-f391-477d-81ac-8694ad7dded6"}
112442024-02-04T14:05:13.218Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:12.246761796Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545}
112452024-02-04T14:05:13.218ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112462024-02-04T14:05:13.218ZLoading write count information from file "/tmp/test_repair_verify.data"
112472024-02-04T14:05:13.218ZRun Repair workload
112482024-02-04T14:05:13.218Z01/30 Flush
112492024-02-04T14:05:13.219Z02/30 Write block 484 len 36864 data: 5 7 7 7 9 12 12 13 14
112502024-02-04T14:05:13.224Z03/30 Read block 547 len 20480
112512024-02-04T14:05:13.225Z04/30 Read block 338 len 36864
112522024-02-04T14:05:13.228Z05/30 Write block 138 len 12288 data: 4 5 9
112532024-02-04T14:05:13.228Z06/30 Read block 554 len 28672
112542024-02-04T14:05:13.228Z07/30 Write block 589 len 4096 data: 7
112552024-02-04T14:05:13.228Z08/30 Read block 340 len 8192
112562024-02-04T14:05:13.228Z09/30 Write block 436 len 20480 data: 10 8 7 5 8
112572024-02-04T14:05:13.228Z10/30 Read block 471 len 36864
112582024-02-04T14:05:13.231Z11/30 Write block 67 len 12288 data: 7 8 6
112592024-02-04T14:05:13.231Z12/30 Read block 290 len 16384
112602024-02-04T14:05:13.234Z13/30 Read block 284 len 36864
112612024-02-04T14:05:13.234Z14/30 Read block 317 len 32768
112622024-02-04T14:05:13.237Z15/30 Read block 111 len 24576
112632024-02-04T14:05:13.239Z16/30 Read block 236 len 8192
112642024-02-04T14:05:13.239Z17/30 Write block 114 len 8192 data: 12 12
112652024-02-04T14:05:13.240Z18/30 Read block 170 len 40960
112662024-02-04T14:05:13.242Z19/30 Write block 362 len 36864 data: 8 9 8 8 8 10 9 9 10
112672024-02-04T14:05:13.242Z20/30 Read block 514 len 4096
112682024-02-04T14:05:13.245Z21/30 Read block 519 len 24576
112692024-02-04T14:05:13.248Z22/30 Write block 254 len 36864 data: 11 12 11 9 9 9 7 9 10
112702024-02-04T14:05:13.248Z23/30 Read block 557 len 24576
112712024-02-04T14:05:13.251Z24/30 Read block 485 len 40960
112722024-02-04T14:05:13.251Z25/30 Read block 549 len 8192
112732024-02-04T14:05:13.254Z26/30 Write block 581 len 40960 data: 12 8 10 12 11 10 9 10 8 8
112742024-02-04T14:05:13.254Z27/30 Read block 204 len 8192
112752024-02-04T14:05:13.257Z28/30 Read block 219 len 8192
112762024-02-04T14:05:13.257Z29/30 Read block 377 len 20480
112772024-02-04T14:05:13.260Z30/30 Read block 116 len 8192
112782024-02-04T14:05:13.260Z----------------------------------------------------------------
112792024-02-04T14:05:13.260Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:29
112802024-02-04T14:05:13.260ZStates: Active Active Active
112812024-02-04T14:05:13.260ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112822024-02-04T14:05:13.260Z 2 Acked 1001 Write 9 Sent Done Done false
112832024-02-04T14:05:13.260Z 3 Acked 1002 Read 5 Sent Done Done false
112842024-02-04T14:05:13.260Z 4 Acked 1003 Read 9 Sent Done Done false
112852024-02-04T14:05:13.260Z 5 Acked 1004 Write 3 Sent Done Done false
112862024-02-04T14:05:13.260Z 6 Acked 1005 Read 7 Sent Done Done false
112872024-02-04T14:05:13.260Z 7 Acked 1006 Write 1 Sent Done Done false
112882024-02-04T14:05:13.260Z 8 Acked 1007 Read 2 Sent Done Done false
112892024-02-04T14:05:13.261Z 9 Acked 1008 Write 5 Sent Done Done false
112902024-02-04T14:05:13.261Z 10 Acked 1009 Read 9 Sent Done Done false
112912024-02-04T14:05:13.261Z 11 Acked 1010 Write 3 Sent Done Done false
112922024-02-04T14:05:13.261Z 12 Acked 1011 Read 4 Sent Done Done false
112932024-02-04T14:05:13.261Z 13 Acked 1012 Read 9 Sent Done Done false
112942024-02-04T14:05:13.261Z 14 Acked 1013 Read 8 Sent Done Done false
112952024-02-04T14:05:13.261Z 15 Acked 1014 Read 6 Sent Done Done false
112962024-02-04T14:05:13.261Z 16 Acked 1015 Read 2 Sent Done Done false
112972024-02-04T14:05:13.261Z 17 Acked 1016 Write 2 Sent Done Done false
112982024-02-04T14:05:13.261Z 18 Acked 1017 Read 10 Sent Done Done false
112992024-02-04T14:05:13.261Z 19 Acked 1018 Write 9 Sent Done Done false
113002024-02-04T14:05:13.261Z 20 Acked 1019 Read 1 Sent Done Done false
113012024-02-04T14:05:13.261Z 21 Acked 1020 Read 6 Sent Done Done false
113022024-02-04T14:05:13.261Z 22 Acked 1021 Write 9 Sent Done Done false
113032024-02-04T14:05:13.261Z 23 Acked 1022 Read 6 Sent Done Done false
113042024-02-04T14:05:13.261Z 24 Acked 1023 Read 10 Sent Done Done false
113052024-02-04T14:05:13.261Z 25 Acked 1024 Read 2 Sent Done Done false
113062024-02-04T14:05:13.261Z 26 Acked 1025 Write 10 Sent Done Done false
113072024-02-04T14:05:13.261Z 27 Acked 1026 Read 2 Sent Done Done false
113082024-02-04T14:05:13.261Z 28 Acked 1027 Read 2 Sent Done Done false
113092024-02-04T14:05:13.261Z 29 Acked 1028 Read 5 Sent Done Done false
113102024-02-04T14:05:13.261Z 30 Acked 1029 Read 2 Sent Sent Done false
113112024-02-04T14:05:13.261Z STATES DS:0 DS:1 DS:2 TOTAL
113122024-02-04T14:05:13.261Z New 0 0 0 0
113132024-02-04T14:05:13.262Z Sent 29 1 0 30
113142024-02-04T14:05:13.262Z Done 0 28 29 57
113152024-02-04T14:05:13.262Z Skipped 0 0 0 0
113162024-02-04T14:05:13.262Z Error 0 0 0 0
113172024-02-04T14:05:13.262ZLast Flush: 1000 1000 1000
113182024-02-04T14:05:13.262ZDownstairs last five completed: 1000
113192024-02-04T14:05:13.262ZUpstairs last five completed: 30 29 28 27 26
113202024-02-04T14:05:13.262Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:12.29161586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1545}
113212024-02-04T14:05:13.262ZWrote out file "/tmp/test_repair_verify.data"
113222024-02-04T14:05:13.265Z
113232024-02-04T14:05:13.268ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113242024-02-04T14:05:13.299ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113252024-02-04T14:05:13.299Z 0 000-019 87 87 87 139 139 139 F F F
113262024-02-04T14:05:13.299Z 1 020-039 102 102 102 171 171 171 F F F
113272024-02-04T14:05:13.299Z 2 040-059 103 103 103 172 172 172 F F F
113282024-02-04T14:05:13.299Z 3 060-079 100 100 100 164 164 164 F T T <---
113292024-02-04T14:05:13.299Z 4 080-099 97 97 97 158 158 158 F F F
113302024-02-04T14:05:13.299Z 5 100-119 92 92 92 150 150 150 F T T <---
113312024-02-04T14:05:13.299Z 6 120-139 100 100 100 164 164 164 F T T <---
113322024-02-04T14:05:13.300Z 7 140-159 100 100 100 164 164 164 F T T <---
113332024-02-04T14:05:13.303Z 8 160-179 102 102 102 171 171 171 F F F
113342024-02-04T14:05:13.303Z 9 180-199 101 101 101 168 168 168 F F F
113352024-02-04T14:05:13.303Z 10 200-219 102 102 102 170 170 170 F F F
113362024-02-04T14:05:13.303Z 11 220-239 98 98 98 160 160 160 F F F
113372024-02-04T14:05:13.303Z 12 240-259 102 102 102 171 171 171 F T T <---
113382024-02-04T14:05:13.303Z 13 260-279 102 102 102 169 169 169 F T T <---
113392024-02-04T14:05:13.303Z 14 280-299 102 102 102 171 171 171 F F F
113402024-02-04T14:05:13.303Z 15 300-319 102 102 102 171 171 171 F F F
113412024-02-04T14:05:13.303Z 16 320-339 86 86 86 137 137 137 F F F
113422024-02-04T14:05:13.303Z 17 340-359 102 102 102 171 171 171 F F F
113432024-02-04T14:05:13.303Z 18 360-379 90 90 90 145 145 145 F T T <---
113442024-02-04T14:05:13.303Z 19 380-399 97 97 97 158 158 158 F F F
113452024-02-04T14:05:13.303Z 20 400-419 100 100 100 167 167 167 F F F
113462024-02-04T14:05:13.303Z 21 420-439 103 103 103 172 172 172 F T T <---
113472024-02-04T14:05:13.303Z 22 440-459 103 103 103 172 172 172 F T T <---
113482024-02-04T14:05:13.303Z 23 460-479 85 85 85 135 135 135 F F F
113492024-02-04T14:05:13.303Z 24 480-499 98 98 98 161 161 161 F T T <---
113502024-02-04T14:05:13.303Z 25 500-519 96 96 96 157 157 157 F F F
113512024-02-04T14:05:13.303Z 26 520-539 103 103 103 172 172 172 F F F
113522024-02-04T14:05:13.304Z 27 540-559 100 100 100 165 165 165 F F F
113532024-02-04T14:05:13.304Z 28 560-579 100 100 100 165 165 165 F F F
113542024-02-04T14:05:13.304Z 29 580-599 100 100 100 167 167 167 F T T <---
113552024-02-04T14:05:13.304ZMax gen: 103, Max flush: 172
113562024-02-04T14:05:13.304ZError: Difference in extent metadata found!
113572024-02-04T14:05:13.304ZA Difference in extent metadata is expected here
113582024-02-04T14:05:13.304ZOn loop 52
113592024-02-04T14:05:13.304Z
113602024-02-04T14:05:13.307ZVerifying data now
113612024-02-04T14:05:18.293ZNo change after verify
113622024-02-04T14:05:18.293ZLoop: 52 Downstairs dump after verify (and repair):
113632024-02-04T14:05:18.316ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113642024-02-04T14:05:18.316Z 0 000-019 87 87 87 139 139 139 F F F
113652024-02-04T14:05:18.319Z 1 020-039 102 102 102 171 171 171 F F F
113662024-02-04T14:05:18.319Z 2 040-059 103 103 103 172 172 172 F F F
113672024-02-04T14:05:18.319Z 3 060-079 104 104 104 173 173 173 F F F
113682024-02-04T14:05:18.319Z 4 080-099 97 97 97 158 158 158 F F F
113692024-02-04T14:05:18.320Z 5 100-119 104 104 104 173 173 173 F F F
113702024-02-04T14:05:18.320Z 6 120-139 104 104 104 173 173 173 F F F
113712024-02-04T14:05:18.320Z 7 140-159 104 104 104 173 173 173 F F F
113722024-02-04T14:05:18.320Z 8 160-179 102 102 102 171 171 171 F F F
113732024-02-04T14:05:18.320Z 9 180-199 101 101 101 168 168 168 F F F
113742024-02-04T14:05:18.320Z 10 200-219 102 102 102 170 170 170 F F F
113752024-02-04T14:05:18.320Z 11 220-239 98 98 98 160 160 160 F F F
113762024-02-04T14:05:18.320Z 12 240-259 104 104 104 173 173 173 F F F
113772024-02-04T14:05:18.320Z 13 260-279 104 104 104 173 173 173 F F F
113782024-02-04T14:05:18.320Z 14 280-299 102 102 102 171 171 171 F F F
113792024-02-04T14:05:18.320Z 15 300-319 102 102 102 171 171 171 F F F
113802024-02-04T14:05:18.320Z 16 320-339 86 86 86 137 137 137 F F F
113812024-02-04T14:05:18.321Z 17 340-359 102 102 102 171 171 171 F F F
113822024-02-04T14:05:18.321Z 18 360-379 104 104 104 173 173 173 F F F
113832024-02-04T14:05:18.321Z 19 380-399 97 97 97 158 158 158 F F F
113842024-02-04T14:05:18.321Z 20 400-419 100 100 100 167 167 167 F F F
113852024-02-04T14:05:18.321Z 21 420-439 104 104 104 173 173 173 F F F
113862024-02-04T14:05:18.321Z 22 440-459 104 104 104 173 173 173 F F F
113872024-02-04T14:05:18.321Z 23 460-479 85 85 85 135 135 135 F F F
113882024-02-04T14:05:18.321Z 24 480-499 104 104 104 173 173 173 F F F
113892024-02-04T14:05:18.321Z 25 500-519 96 96 96 157 157 157 F F F
113902024-02-04T14:05:18.321Z 26 520-539 103 103 103 172 172 172 F F F
113912024-02-04T14:05:18.321Z 27 540-559 100 100 100 165 165 165 F F F
113922024-02-04T14:05:18.321Z 28 560-579 100 100 100 165 165 165 F F F
113932024-02-04T14:05:18.321Z 29 580-599 104 104 104 173 173 173 F F F
113942024-02-04T14:05:18.322ZMax gen: 104, Max flush: 173
113952024-02-04T14:05:18.325Z
113962024-02-04T14:05:18.325ZBegin loop 53
113972024-02-04T14:05:18.325ZDownstairs to restart: 1
113982024-02-04T14:05:18.350Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.383298386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553}
113992024-02-04T14:05:18.350Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.383780305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553}
114002024-02-04T14:05:18.354Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.38385096Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553}
114012024-02-04T14:05:18.355Z{"msg":"Crucible ed787044-9345-45f5-ad1e-7c66e3b01dd2 has session id: 849b1733-0acd-4b44-a4af-940487087b21","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.383978037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114022024-02-04T14:05:18.355ZCrucible runtime is spawned
114032024-02-04T14:05:18.355Z{"msg":"Upstairs opts: Upstairs UUID: ed787044-9345-45f5-ad1e-7c66e3b01dd2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384177337Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114042024-02-04T14:05:18.355Z{"msg":"Crucible stats registered with UUID: ed787044-9345-45f5-ad1e-7c66e3b01dd2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384252328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114052024-02-04T14:05:18.355Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384321316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553}
114062024-02-04T14:05:18.355Z{"msg":"ed787044-9345-45f5-ad1e-7c66e3b01dd2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384490954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114072024-02-04T14:05:18.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.38455661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114082024-02-04T14:05:18.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384607336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114092024-02-04T14:05:18.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384632688Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114102024-02-04T14:05:18.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384655586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114112024-02-04T14:05:18.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384678393Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114122024-02-04T14:05:18.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384735014Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114132024-02-04T14:05:18.356Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384771189Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114142024-02-04T14:05:18.356Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384838708Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114152024-02-04T14:05:18.356Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.384883698Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114162024-02-04T14:05:18.356Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.385139222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114172024-02-04T14:05:18.356Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.385238499Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114182024-02-04T14:05:18.356Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.385283371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114192024-02-04T14:05:18.356Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.386727476Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114202024-02-04T14:05:18.356Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.38684Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114212024-02-04T14:05:18.356Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.386870765Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114222024-02-04T14:05:18.356Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.387047286Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114232024-02-04T14:05:18.356Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.387112885Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114242024-02-04T14:05:18.356Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.387139209Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114252024-02-04T14:05:18.361Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.387838841Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114262024-02-04T14:05:18.361Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.387999254Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114272024-02-04T14:05:18.361Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.388462697Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114282024-02-04T14:05:18.361Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.388827695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114292024-02-04T14:05:18.361Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.388897852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114302024-02-04T14:05:18.362Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.388926122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114312024-02-04T14:05:18.362Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.388947643Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114322024-02-04T14:05:18.362Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.388972405Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114332024-02-04T14:05:18.362Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389015184Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114342024-02-04T14:05:18.362Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389317822Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114352024-02-04T14:05:18.362Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389399345Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114362024-02-04T14:05:18.362Z{"msg":"[0]R flush_numbers[0..12]: [139, 171, 172, 173, 158, 173, 173, 173, 171, 168, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389438069Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114372024-02-04T14:05:18.362Z{"msg":"[0]R generation[0..12]: [87, 102, 103, 104, 97, 104, 104, 104, 102, 101, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389496487Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114382024-02-04T14:05:18.362Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389529183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114392024-02-04T14:05:18.362ZWait for a query_work_queue command to finish before sending IO
114402024-02-04T14:05:18.362Z{"msg":"[1]R flush_numbers[0..12]: [139, 171, 172, 173, 158, 173, 173, 173, 171, 168, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389589686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114412024-02-04T14:05:18.362Z{"msg":"[1]R generation[0..12]: [87, 102, 103, 104, 97, 104, 104, 104, 102, 101, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389653706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114422024-02-04T14:05:18.363Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389720728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114432024-02-04T14:05:18.363Z{"msg":"[2]R flush_numbers[0..12]: [139, 171, 172, 173, 158, 173, 173, 173, 171, 168, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389797864Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114442024-02-04T14:05:18.363Z{"msg":"[2]R generation[0..12]: [87, 102, 103, 104, 97, 104, 104, 104, 102, 101, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.38982864Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114452024-02-04T14:05:18.363Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389853859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114462024-02-04T14:05:18.363Z{"msg":"Max found gen is 105","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389903936Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114472024-02-04T14:05:18.363Z{"msg":"Generation requested: 106 >= found:105","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389928234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114482024-02-04T14:05:18.363Z{"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.389950374Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114492024-02-04T14:05:18.363Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.38999524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114502024-02-04T14:05:18.363Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.390020484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114512024-02-04T14:05:18.363Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.390041327Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114522024-02-04T14:05:18.363Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.390063439Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114532024-02-04T14:05:18.363Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.39009246Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114542024-02-04T14:05:18.364Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.390128536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114552024-02-04T14:05:18.364Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.390160204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114562024-02-04T14:05:18.364Z{"msg":"ed787044-9345-45f5-ad1e-7c66e3b01dd2 is now active with session: 849b1733-0acd-4b44-a4af-940487087b21","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.390194138Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114572024-02-04T14:05:18.364Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.390220702Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553,"session_id":"849b1733-0acd-4b44-a4af-940487087b21"}
114582024-02-04T14:05:18.364Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:17.390243138Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553}
114592024-02-04T14:05:18.364ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114602024-02-04T14:05:18.364ZLoading write count information from file "/tmp/test_repair_verify.data"
114612024-02-04T14:05:18.364ZRun Repair workload
114622024-02-04T14:05:18.364Z01/30 Write block 357 len 24576 data: 7 6 6 6 8 9
114632024-02-04T14:05:18.368Z02/30 Read block 518 len 20480
114642024-02-04T14:05:18.368Z03/30 Write block 575 len 32768 data: 9 12 12 11 13 14 13 9
114652024-02-04T14:05:18.368Z04/30 Read block 266 len 20480
114662024-02-04T14:05:18.368Z05/30 Write block 125 len 4096 data: 9
114672024-02-04T14:05:18.368Z06/30 Read block 396 len 16384
114682024-02-04T14:05:18.383Z07/30 Write block 178 len 40960 data: 9 12 11 13 11 14 12 13 11 11
114692024-02-04T14:05:18.383Z08/30 Flush
114702024-02-04T14:05:18.388Z09/30 Write block 30 len 8192 data: 7 8
114712024-02-04T14:05:18.388Z10/30 Write block 580 len 40960 data: 15 14 10 11 13 12 11 10 11 9
114722024-02-04T14:05:18.388Z11/30 Read block 412 len 32768
114732024-02-04T14:05:18.388Z12/30 Write block 273 len 32768 data: 7 6 6 5 5 7 7 7
114742024-02-04T14:05:18.388Z13/30 Read block 298 len 4096
114752024-02-04T14:05:18.388Z14/30 Write block 340 len 40960 data: 7 7 6 7 8 9 8 9 9 11
114762024-02-04T14:05:18.388Z15/30 Read block 554 len 20480
114772024-02-04T14:05:18.388Z16/30 Read block 320 len 12288
114782024-02-04T14:05:18.388Z17/30 Read block 4 len 16384
114792024-02-04T14:05:18.392Z18/30 Read block 332 len 16384
114802024-02-04T14:05:18.392Z19/30 Read block 208 len 4096
114812024-02-04T14:05:18.392Z20/30 Read block 390 len 20480
114822024-02-04T14:05:18.392Z21/30 Write block 135 len 24576 data: 6 4 4 5 6 10
114832024-02-04T14:05:18.395Z22/30 Write block 63 len 28672 data: 8 8 7 5 8 9 7
114842024-02-04T14:05:18.395Z23/30 Write block 302 len 40960 data: 7 6 7 9 8 8 6 7 8 8
114852024-02-04T14:05:18.395Z24/30 Read block 267 len 12288
114862024-02-04T14:05:18.398Z25/30 Flush
114872024-02-04T14:05:18.402Z26/30 Write block 408 len 28672 data: 9 10 10 11 10 10 11
114882024-02-04T14:05:18.405Z27/30 Read block 322 len 20480
114892024-02-04T14:05:18.408Z28/30 Write block 553 len 28672 data: 6 6 6 8 9 8 10
114902024-02-04T14:05:18.408Z29/30 Read block 533 len 24576
114912024-02-04T14:05:18.408Z30/30 Read block 122 len 16384
114922024-02-04T14:05:18.412Z----------------------------------------------------------------
114932024-02-04T14:05:18.412Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
114942024-02-04T14:05:18.412ZStates: Active Active Active
114952024-02-04T14:05:18.412ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114962024-02-04T14:05:18.412Z 1 Acked 1000 Write 6 Done Done Done false
114972024-02-04T14:05:18.412Z 2 Acked 1001 Read 5 Done Sent Done false
114982024-02-04T14:05:18.412Z 3 Acked 1002 Write 8 Done Sent Done false
114992024-02-04T14:05:18.412Z 4 Acked 1003 Read 5 Done Sent Done false
115002024-02-04T14:05:18.412Z 5 Acked 1004 Write 1 Done Sent Done false
115012024-02-04T14:05:18.413Z 6 Acked 1005 Read 4 Done Sent Done false
115022024-02-04T14:05:18.413Z 7 Acked 1006 Write 10 Done Sent Done false
115032024-02-04T14:05:18.413Z 8 Acked 1007 Flush 0 Done Sent Done false
115042024-02-04T14:05:18.413Z 9 Acked 1008 Write 2 Done Sent Done false
115052024-02-04T14:05:18.413Z 10 Acked 1009 Write 10 Done Sent Done false
115062024-02-04T14:05:18.413Z 11 Acked 1010 Read 8 Done Sent Done false
115072024-02-04T14:05:18.413Z 12 Acked 1011 Write 8 Done Sent Done false
115082024-02-04T14:05:18.413Z 13 Acked 1012 Read 1 Done Sent Done false
115092024-02-04T14:05:18.413Z 14 Acked 1013 Write 10 Done Sent Done false
115102024-02-04T14:05:18.413Z 15 Acked 1014 Read 5 Done Sent Done false
115112024-02-04T14:05:18.413Z 16 Acked 1015 Read 3 Done Sent Done false
115122024-02-04T14:05:18.413Z 17 Acked 1016 Read 4 Done Sent Done false
115132024-02-04T14:05:18.413Z 18 Acked 1017 Read 4 Done Sent Done false
115142024-02-04T14:05:18.414Z 19 Acked 1018 Read 1 Done Sent Done false
115152024-02-04T14:05:18.414Z 20 Acked 1019 Read 5 Done Sent Done false
115162024-02-04T14:05:18.414Z 21 Acked 1020 Write 6 Done Sent Done false
115172024-02-04T14:05:18.414Z 22 Acked 1021 Write 7 Done Sent Done false
115182024-02-04T14:05:18.414Z 23 Acked 1022 Write 10 Done Sent Done false
115192024-02-04T14:05:18.414Z 24 Acked 1023 Read 3 Done Sent Done false
115202024-02-04T14:05:18.414Z 25 Acked 1024 Flush 0 Done Sent Done false
115212024-02-04T14:05:18.414Z 26 Acked 1025 Write 7 Done Sent Done false
115222024-02-04T14:05:18.414Z 27 Acked 1026 Read 5 Done Sent Done false
115232024-02-04T14:05:18.414Z 28 Acked 1027 Write 7 Done Sent Done false
115242024-02-04T14:05:18.414Z 29 Acked 1028 Read 6 Done Sent Done false
115252024-02-04T14:05:18.414Z 30 Acked 1029 Read 4 Done Sent Done false
115262024-02-04T14:05:18.415Z STATES DS:0 DS:1 DS:2 TOTAL
115272024-02-04T14:05:18.415Z New 0 0 0 0
115282024-02-04T14:05:18.415Z Sent 0 29 0 29
115292024-02-04T14:05:18.415Z Done 30 1 30 61
115302024-02-04T14:05:18.415Z Skipped 0 0 0 0
115312024-02-04T14:05:18.415Z Error 0 0 0 0
115322024-02-04T14:05:18.415ZLast Flush: 1024 0 1024
115332024-02-04T14:05:18.415ZDownstairs last five completed:
115342024-02-04T14:05:18.415ZUpstairs last five completed: 30 29 28 27 26
115352024-02-04T14:05:18.415Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:17.442454212Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1553}
115362024-02-04T14:05:18.415ZWrote out file "/tmp/test_repair_verify.data"
115372024-02-04T14:05:18.419Z
115382024-02-04T14:05:18.422ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115392024-02-04T14:05:18.445ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115402024-02-04T14:05:18.445Z 0 000-019 87 87 87 139 139 139 F F F
115412024-02-04T14:05:18.445Z 1 020-039 106 102 106 175 171 175 F F F <---
115422024-02-04T14:05:18.445Z 2 040-059 103 103 103 172 172 172 F F F
115432024-02-04T14:05:18.448Z 3 060-079 106 104 106 175 173 175 F F F <---
115442024-02-04T14:05:18.448Z 4 080-099 97 97 97 158 158 158 F F F
115452024-02-04T14:05:18.449Z 5 100-119 104 104 104 173 173 173 F F F
115462024-02-04T14:05:18.449Z 6 120-139 106 104 106 175 173 175 F F F <---
115472024-02-04T14:05:18.449Z 7 140-159 106 104 106 175 173 175 F F F <---
115482024-02-04T14:05:18.449Z 8 160-179 106 102 106 174 171 174 F F F <---
115492024-02-04T14:05:18.449Z 9 180-199 106 101 106 174 168 174 F F F <---
115502024-02-04T14:05:18.449Z 10 200-219 102 102 102 170 170 170 F F F
115512024-02-04T14:05:18.449Z 11 220-239 98 98 98 160 160 160 F F F
115522024-02-04T14:05:18.449Z 12 240-259 104 104 104 173 173 173 F F F
115532024-02-04T14:05:18.449Z 13 260-279 106 104 106 175 173 175 F F F <---
115542024-02-04T14:05:18.449Z 14 280-299 106 102 106 175 171 175 F F F <---
115552024-02-04T14:05:18.449Z 15 300-319 106 102 106 175 171 175 F F F <---
115562024-02-04T14:05:18.449Z 16 320-339 86 86 86 137 137 137 F F F
115572024-02-04T14:05:18.449Z 17 340-359 106 102 106 175 171 175 F T F <---
115582024-02-04T14:05:18.450Z 18 360-379 106 104 106 174 173 174 F T F <---
115592024-02-04T14:05:18.450Z 19 380-399 97 97 97 158 158 158 F F F
115602024-02-04T14:05:18.450Z 20 400-419 100 100 100 167 167 167 T F T <---
115612024-02-04T14:05:18.450Z 21 420-439 104 104 104 173 173 173 F F F
115622024-02-04T14:05:18.450Z 22 440-459 104 104 104 173 173 173 F F F
115632024-02-04T14:05:18.450Z 23 460-479 85 85 85 135 135 135 F F F
115642024-02-04T14:05:18.450Z 24 480-499 104 104 104 173 173 173 F F F
115652024-02-04T14:05:18.450Z 25 500-519 96 96 96 157 157 157 F F F
115662024-02-04T14:05:18.450Z 26 520-539 103 103 103 172 172 172 F F F
115672024-02-04T14:05:18.450Z 27 540-559 100 100 100 165 165 165 T F T <---
115682024-02-04T14:05:18.450Z 28 560-579 106 100 106 174 165 174 F F F <---
115692024-02-04T14:05:18.450Z 29 580-599 106 104 106 175 173 175 F F F <---
115702024-02-04T14:05:18.450ZMax gen: 106, Max flush: 175
115712024-02-04T14:05:18.451ZError: Difference in extent metadata found!
115722024-02-04T14:05:18.454ZA Difference in extent metadata is expected here
115732024-02-04T14:05:18.454ZOn loop 53
115742024-02-04T14:05:18.454Z
115752024-02-04T14:05:18.454ZVerifying data now
115762024-02-04T14:05:24.713ZNo change after verify
115772024-02-04T14:05:24.713ZLoop: 53 Downstairs dump after verify (and repair):
115782024-02-04T14:05:24.738ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115792024-02-04T14:05:24.738Z 0 000-019 87 87 87 139 139 139 F F F
115802024-02-04T14:05:24.738Z 1 020-039 106 106 106 175 175 175 F F F
115812024-02-04T14:05:24.738Z 2 040-059 103 103 103 172 172 172 F F F
115822024-02-04T14:05:24.738Z 3 060-079 106 106 106 175 175 175 F F F
115832024-02-04T14:05:24.739Z 4 080-099 97 97 97 158 158 158 F F F
115842024-02-04T14:05:24.739Z 5 100-119 104 104 104 173 173 173 F F F
115852024-02-04T14:05:24.739Z 6 120-139 106 106 106 175 175 175 F F F
115862024-02-04T14:05:24.739Z 7 140-159 106 106 106 175 175 175 F F F
115872024-02-04T14:05:24.739Z 8 160-179 106 106 106 174 174 174 F F F
115882024-02-04T14:05:24.739Z 9 180-199 106 106 106 174 174 174 F F F
115892024-02-04T14:05:24.739Z 10 200-219 102 102 102 170 170 170 F F F
115902024-02-04T14:05:24.739Z 11 220-239 98 98 98 160 160 160 F F F
115912024-02-04T14:05:24.739Z 12 240-259 104 104 104 173 173 173 F F F
115922024-02-04T14:05:24.739Z 13 260-279 106 106 106 175 175 175 F F F
115932024-02-04T14:05:24.739Z 14 280-299 106 106 106 175 175 175 F F F
115942024-02-04T14:05:24.739Z 15 300-319 106 106 106 175 175 175 F F F
115952024-02-04T14:05:24.740Z 16 320-339 86 86 86 137 137 137 F F F
115962024-02-04T14:05:24.740Z 17 340-359 106 106 106 175 175 175 F F F
115972024-02-04T14:05:24.743Z 18 360-379 106 106 106 174 174 174 F F F
115982024-02-04T14:05:24.743Z 19 380-399 97 97 97 158 158 158 F F F
115992024-02-04T14:05:24.743Z 20 400-419 107 107 107 176 176 176 F F F
116002024-02-04T14:05:24.744Z 21 420-439 104 104 104 173 173 173 F F F
116012024-02-04T14:05:24.744Z 22 440-459 104 104 104 173 173 173 F F F
116022024-02-04T14:05:24.744Z 23 460-479 85 85 85 135 135 135 F F F
116032024-02-04T14:05:24.744Z 24 480-499 104 104 104 173 173 173 F F F
116042024-02-04T14:05:24.744Z 25 500-519 96 96 96 157 157 157 F F F
116052024-02-04T14:05:24.744Z 26 520-539 103 103 103 172 172 172 F F F
116062024-02-04T14:05:24.744Z 27 540-559 107 107 107 176 176 176 F F F
116072024-02-04T14:05:24.744Z 28 560-579 106 106 106 174 174 174 F F F
116082024-02-04T14:05:24.744Z 29 580-599 106 106 106 175 175 175 F F F
116092024-02-04T14:05:24.744ZMax gen: 107, Max flush: 176
116102024-02-04T14:05:24.759Z
116112024-02-04T14:05:24.759ZBegin loop 54
116122024-02-04T14:05:24.760ZDownstairs to restart: 1
116132024-02-04T14:05:24.780Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.810064726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561}
116142024-02-04T14:05:24.780Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81052619Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561}
116152024-02-04T14:05:24.788Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.810575209Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561}
116162024-02-04T14:05:24.788Z{"msg":"Crucible 1bcc4427-35ab-4712-892c-f3adc4169556 has session id: 3ebf8561-a797-420e-8898-2f4cdefab754","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.810651669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116172024-02-04T14:05:24.788Z{"msg":"Upstairs opts: Upstairs UUID: 1bcc4427-35ab-4712-892c-f3adc4169556, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible"Crucible runtime is spawned
116182024-02-04T14:05:24.788Z,"level":30,"time":"2024-02-04T14:05:23.810816685Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116192024-02-04T14:05:24.788Z{"msg":"Crucible stats registered with UUID: 1bcc4427-35ab-4712-892c-f3adc4169556","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.810899623Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116202024-02-04T14:05:24.788Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.810945037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561}
116212024-02-04T14:05:24.788Z{"msg":"1bcc4427-35ab-4712-892c-f3adc4169556 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811008091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116222024-02-04T14:05:24.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811032647Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116232024-02-04T14:05:24.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811107876Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116242024-02-04T14:05:24.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811149638Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116252024-02-04T14:05:24.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811213244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116262024-02-04T14:05:24.789Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811258079Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116272024-02-04T14:05:24.789Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811334825Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116282024-02-04T14:05:24.789Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811519684Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"io task","client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116292024-02-04T14:05:24.789Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811616421Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"io task","client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116302024-02-04T14:05:24.789Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811692992Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"io task","client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116312024-02-04T14:05:24.789Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811743187Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"io task","client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116322024-02-04T14:05:24.789Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811817808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"io task","client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116332024-02-04T14:05:24.789Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.811865585Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"io task","client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116342024-02-04T14:05:24.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.812601717Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116352024-02-04T14:05:24.789Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.812701571Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116362024-02-04T14:05:24.789Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81272828Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116372024-02-04T14:05:24.790Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81278519Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116382024-02-04T14:05:24.790Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.813625258Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116392024-02-04T14:05:24.790Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.813888794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116402024-02-04T14:05:24.790Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.814555854Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116412024-02-04T14:05:24.790Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.814687521Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116422024-02-04T14:05:24.790Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.814767863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116432024-02-04T14:05:24.790Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.814795136Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116442024-02-04T14:05:24.790Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.814818522Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116452024-02-04T14:05:24.790Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.814873304Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116462024-02-04T14:05:24.790Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.814931165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116472024-02-04T14:05:24.790Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.814981797Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116482024-02-04T14:05:24.790Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.815760319Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116492024-02-04T14:05:24.791Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.816487433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116502024-02-04T14:05:24.791Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81657677Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116512024-02-04T14:05:24.795Z{"msg":"[0]R flush_numbers[0..12]: [139, 175, 172, 175, 158, 173, 175, 175, 174, 174, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.816655447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116522024-02-04T14:05:24.795Z{"msg":"[0]R generation[0..12]: [87, 106, 103, 106, 97, 104, 106, 106, 106, 106, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.816732308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116532024-02-04T14:05:24.795Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.816779442Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116542024-02-04T14:05:24.796Z{"msg":"[1]R flush_numbers[0..12]: [139, 175, 172, 175, 158, 173, 175, 175, 174, 174, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.816825326Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116552024-02-04T14:05:24.796ZWait for a query_work_queue command to finish before sending IO
116562024-02-04T14:05:24.796Z{"msg":"[1]R generation[0..12]: [87, 106, 103, 106, 97, 104, 106, 106, 106, 106, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.816902788Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116572024-02-04T14:05:24.796Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81698519Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116582024-02-04T14:05:24.796Z{"msg":"[2]R flush_numbers[0..12]: [139, 175, 172, 175, 158, 173, 175, 175, 174, 174, 170, 160]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817062343Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116592024-02-04T14:05:24.796Z{"msg":"[2]R generation[0..12]: [87, 106, 103, 106, 97, 104, 106, 106, 106, 106, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817137236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116602024-02-04T14:05:24.796Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81721287Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116612024-02-04T14:05:24.796Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817289051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116622024-02-04T14:05:24.796Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817325007Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116632024-02-04T14:05:24.796Z{"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81734768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116642024-02-04T14:05:24.796Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817369662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116652024-02-04T14:05:24.796Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817391999Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116662024-02-04T14:05:24.796Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81741744Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116672024-02-04T14:05:24.797Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.81745763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116682024-02-04T14:05:24.797Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817498506Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116692024-02-04T14:05:24.797Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817568039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116702024-02-04T14:05:24.797Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817610348Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116712024-02-04T14:05:24.797Z{"msg":"1bcc4427-35ab-4712-892c-f3adc4169556 is now active with session: 3ebf8561-a797-420e-8898-2f4cdefab754","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817680948Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116722024-02-04T14:05:24.797Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:23.817755712Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561,"session_id":"3ebf8561-a797-420e-8898-2f4cdefab754"}
116732024-02-04T14:05:24.797Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116742024-02-04T14:05:24.797Z,"time":"2024-02-04T14:05:23.817825641Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561}
116752024-02-04T14:05:24.797ZLoading write count information from file "/tmp/test_repair_verify.data"
116762024-02-04T14:05:24.797ZRun Repair workload
116772024-02-04T14:05:24.797Z01/30 Write block 85 len 16384 data: 10 8 6 5
116782024-02-04T14:05:24.797Z02/30 Flush
116792024-02-04T14:05:24.797Z03/30 Write block 228 len 16384 data: 8 7 6 5
116802024-02-04T14:05:24.798Z04/30 Write block 233 len 32768 data: 7 8 6 6 9 9 8 9
116812024-02-04T14:05:24.798Z05/30 Read block 551 len 36864
116822024-02-04T14:05:24.801Z06/30 Write block 553 len 20480 data: 7 7 7 9 10
116832024-02-04T14:05:24.802Z07/30 Read block 511 len 12288
116842024-02-04T14:05:24.802Z08/30 Read block 568 len 32768
116852024-02-04T14:05:24.802Z09/30 Read block 161 len 8192
116862024-02-04T14:05:24.805Z10/30 Read block 455 len 36864
116872024-02-04T14:05:24.805Z11/30 Write block 236 len 4096 data: 7
116882024-02-04T14:05:24.805Z12/30 Write block 485 len 8192 data: 8 8
116892024-02-04T14:05:24.808Z13/30 Write block 165 len 4096 data: 10
116902024-02-04T14:05:24.809Z14/30 Read block 520 len 36864
116912024-02-04T14:05:24.809Z15/30 Write block 504 len 32768 data: 12 9 13 13 12 12 10 9
116922024-02-04T14:05:24.813Z16/30 Write block 193 len 36864 data: 6 4 4 5 8 10 11 13 13
116932024-02-04T14:05:24.813Z17/30 Read block 152 len 32768
116942024-02-04T14:05:24.816Z18/30 Write block 507 len 20480 data: 14 13 13 11 10
116952024-02-04T14:05:24.816Z19/30 Write block 62 len 24576 data: 9 9 9 8 6 9
116962024-02-04T14:05:24.816Z20/30 Read block 570 len 24576
116972024-02-04T14:05:24.819Z21/30 Read block 175 len 40960
116982024-02-04T14:05:24.819Z22/30 Read block 207 len 28672
116992024-02-04T14:05:24.823Z23/30 Write block 313 len 40960 data: 7 6 4 4 6 7 7 7 6 6
117002024-02-04T14:05:24.823Z24/30 Read block 11 len 4096
117012024-02-04T14:05:24.827Z25/30 Read block 214 len 32768
117022024-02-04T14:05:24.827Z26/30 Write block 259 len 32768 data: 10 8 10 11 10 9 9 10
117032024-02-04T14:05:24.827Z27/30 Read block 390 len 28672
117042024-02-04T14:05:24.830Z28/30 Read block 568 len 8192
117052024-02-04T14:05:24.838Z29/30 Write block 177 len 16384 data: 11 10 13 12
117062024-02-04T14:05:24.838Z30/30 Write block 98 len 8192 data: 6 8
117072024-02-04T14:05:24.838Z----------------------------------------------------------------
117082024-02-04T14:05:24.838Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:28
117092024-02-04T14:05:24.838ZStates: Active Active Active
117102024-02-04T14:05:24.838ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117112024-02-04T14:05:24.838Z 3 Acked 1002 Write 4 Done Done Done false
117122024-02-04T14:05:24.838Z 4 Acked 1003 Write 8 Done Done Done false
117132024-02-04T14:05:24.838Z 5 Acked 1004 Read 9 Done Done Done false
117142024-02-04T14:05:24.839Z 6 Acked 1005 Write 5 Done Done Done false
117152024-02-04T14:05:24.839Z 7 Acked 1006 Read 3 Done Done Done false
117162024-02-04T14:05:24.839Z 8 Acked 1007 Read 8 Done Done Done false
117172024-02-04T14:05:24.839Z 9 Acked 1008 Read 2 Done Done Done false
117182024-02-04T14:05:24.839Z 10 Acked 1009 Read 9 Done Done Done false
117192024-02-04T14:05:24.839Z 11 Acked 1010 Write 1 Done Done Done false
117202024-02-04T14:05:24.839Z 12 Acked 1011 Write 2 Done Done Done false
117212024-02-04T14:05:24.839Z 13 Acked 1012 Write 1 Done Sent Done false
117222024-02-04T14:05:24.839Z 14 Acked 1013 Read 9 Done Sent Done false
117232024-02-04T14:05:24.839Z 15 Acked 1014 Write 8 Done Sent Done false
117242024-02-04T14:05:24.839Z 16 Acked 1015 Write 9 Done Sent Done false
117252024-02-04T14:05:24.839Z 17 Acked 1016 Read 8 Done Sent Done false
117262024-02-04T14:05:24.839Z 18 Acked 1017 Write 5 Done Sent Done false
117272024-02-04T14:05:24.840Z 19 Acked 1018 Write 6 Done Sent Done false
117282024-02-04T14:05:24.840Z 20 Acked 1019 Read 6 Done Sent Done false
117292024-02-04T14:05:24.840Z 21 Acked 1020 Read 10 Done Sent Done false
117302024-02-04T14:05:24.840Z 22 Acked 1021 Read 7 Done Sent Done false
117312024-02-04T14:05:24.840Z 23 Acked 1022 Write 10 Done Sent Done false
117322024-02-04T14:05:24.840Z 24 Acked 1023 Read 1 Done Sent Done false
117332024-02-04T14:05:24.840Z 25 Acked 1024 Read 8 Done Sent Done false
117342024-02-04T14:05:24.840Z 26 Acked 1025 Write 8 Done Sent Done false
117352024-02-04T14:05:24.840Z 27 Acked 1026 Read 7 Done Sent Done false
117362024-02-04T14:05:24.840Z 28 Acked 1027 Read 2 Done Sent Done false
117372024-02-04T14:05:24.840Z 29 Acked 1028 Write 4 Sent Sent Sent false
117382024-02-04T14:05:24.840Z 30 Acked 1029 Write 2 Sent Sent Sent false
117392024-02-04T14:05:24.840Z STATES DS:0 DS:1 DS:2 TOTAL
117402024-02-04T14:05:24.841Z New 0 0 0 0
117412024-02-04T14:05:24.841Z Sent 2 18 2 22
117422024-02-04T14:05:24.841Z Done 26 10 26 62
117432024-02-04T14:05:24.841Z Skipped 0 0 0 0
117442024-02-04T14:05:24.841Z Error 0 0 0 0
117452024-02-04T14:05:24.841ZLast Flush: 1001 1001 1001
117462024-02-04T14:05:24.841ZDownstairs last five completed: 1001 1000
117472024-02-04T14:05:24.841ZUpstairs last five completed: 30 29 28 27 26
117482024-02-04T14:05:24.841Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:23.865798826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1561}
117492024-02-04T14:05:24.841ZWrote out file "/tmp/test_repair_verify.data"
117502024-02-04T14:05:24.845Z
117512024-02-04T14:05:24.848ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117522024-02-04T14:05:24.879ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117532024-02-04T14:05:24.879Z 0 000-019 87 87 87 139 139 139 F F F
117542024-02-04T14:05:24.879Z 1 020-039 106 106 106 175 175 175 F F F
117552024-02-04T14:05:24.879Z 2 040-059 103 103 103 172 172 172 F F F
117562024-02-04T14:05:24.879Z 3 060-079 106 106 106 175 175 175 T F T <---
117572024-02-04T14:05:24.879Z 4 080-099 108 108 108 177 177 177 T F T <---
117582024-02-04T14:05:24.879Z 5 100-119 104 104 104 173 173 173 F F F
117592024-02-04T14:05:24.880Z 6 120-139 106 106 106 175 175 175 F F F
117602024-02-04T14:05:24.880Z 7 140-159 106 106 106 175 175 175 F F F
117612024-02-04T14:05:24.880Z 8 160-179 106 106 106 174 174 174 T F T <---
117622024-02-04T14:05:24.880Z 9 180-199 106 106 106 174 174 174 T F T <---
117632024-02-04T14:05:24.880Z 10 200-219 102 102 102 170 170 170 T F T <---
117642024-02-04T14:05:24.880Z 11 220-239 98 98 98 160 160 160 T T T
117652024-02-04T14:05:24.880Z 12 240-259 104 104 104 173 173 173 T T T
117662024-02-04T14:05:24.880Z 13 260-279 106 106 106 175 175 175 T F T <---
117672024-02-04T14:05:24.883Z 14 280-299 106 106 106 175 175 175 F F F
117682024-02-04T14:05:24.883Z 15 300-319 106 106 106 175 175 175 T F T <---
117692024-02-04T14:05:24.884Z 16 320-339 86 86 86 137 137 137 T F T <---
117702024-02-04T14:05:24.884Z 17 340-359 106 106 106 175 175 175 F F F
117712024-02-04T14:05:24.884Z 18 360-379 106 106 106 174 174 174 F F F
117722024-02-04T14:05:24.884Z 19 380-399 97 97 97 158 158 158 F F F
117732024-02-04T14:05:24.884Z 20 400-419 107 107 107 176 176 176 F F F
117742024-02-04T14:05:24.884Z 21 420-439 104 104 104 173 173 173 F F F
117752024-02-04T14:05:24.884Z 22 440-459 104 104 104 173 173 173 F F F
117762024-02-04T14:05:24.884Z 23 460-479 85 85 85 135 135 135 F F F
117772024-02-04T14:05:24.884Z 24 480-499 104 104 104 173 173 173 T T T
117782024-02-04T14:05:24.884Z 25 500-519 96 96 96 157 157 157 T F T <---
117792024-02-04T14:05:24.884Z 26 520-539 103 103 103 172 172 172 F F F
117802024-02-04T14:05:24.884Z 27 540-559 107 107 107 176 176 176 T T T
117812024-02-04T14:05:24.884Z 28 560-579 106 106 106 174 174 174 F F F
117822024-02-04T14:05:24.885Z 29 580-599 106 106 106 175 175 175 F F F
117832024-02-04T14:05:24.885ZMax gen: 108, Max flush: 177
117842024-02-04T14:05:24.885ZError: Difference in extent metadata found!
117852024-02-04T14:05:24.888ZA Difference in extent metadata is expected here
117862024-02-04T14:05:24.888ZOn loop 54
117872024-02-04T14:05:24.888Z
117882024-02-04T14:05:24.914ZVerifying data now
117892024-02-04T14:05:31.065ZNo change after verify
117902024-02-04T14:05:31.066ZLoop: 54 Downstairs dump after verify (and repair):
117912024-02-04T14:05:31.088ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117922024-02-04T14:05:31.089Z 0 000-019 87 87 87 139 139 139 F F F
117932024-02-04T14:05:31.089Z 1 020-039 106 106 106 175 175 175 F F F
117942024-02-04T14:05:31.089Z 2 040-059 103 103 103 172 172 172 F F F
117952024-02-04T14:05:31.089Z 3 060-079 109 109 109 178 178 178 F F F
117962024-02-04T14:05:31.089Z 4 080-099 109 109 109 178 178 178 F F F
117972024-02-04T14:05:31.089Z 5 100-119 104 104 104 173 173 173 F F F
117982024-02-04T14:05:31.089Z 6 120-139 106 106 106 175 175 175 F F F
117992024-02-04T14:05:31.089Z 7 140-159 106 106 106 175 175 175 F F F
118002024-02-04T14:05:31.089Z 8 160-179 109 109 109 178 178 178 F F F
118012024-02-04T14:05:31.089Z 9 180-199 109 109 109 178 178 178 F F F
118022024-02-04T14:05:31.089Z 10 200-219 109 109 109 178 178 178 F F F
118032024-02-04T14:05:31.089Z 11 220-239 109 109 109 178 178 178 F F F
118042024-02-04T14:05:31.092Z 12 240-259 109 109 109 178 178 178 F F F
118052024-02-04T14:05:31.092Z 13 260-279 109 109 109 178 178 178 F F F
118062024-02-04T14:05:31.093Z 14 280-299 106 106 106 175 175 175 F F F
118072024-02-04T14:05:31.093Z 15 300-319 109 109 109 178 178 178 F F F
118082024-02-04T14:05:31.093Z 16 320-339 109 109 109 178 178 178 F F F
118092024-02-04T14:05:31.093Z 17 340-359 106 106 106 175 175 175 F F F
118102024-02-04T14:05:31.093Z 18 360-379 106 106 106 174 174 174 F F F
118112024-02-04T14:05:31.093Z 19 380-399 97 97 97 158 158 158 F F F
118122024-02-04T14:05:31.093Z 20 400-419 107 107 107 176 176 176 F F F
118132024-02-04T14:05:31.093Z 21 420-439 104 104 104 173 173 173 F F F
118142024-02-04T14:05:31.093Z 22 440-459 104 104 104 173 173 173 F F F
118152024-02-04T14:05:31.093Z 23 460-479 85 85 85 135 135 135 F F F
118162024-02-04T14:05:31.093Z 24 480-499 109 109 109 178 178 178 F F F
118172024-02-04T14:05:31.093Z 25 500-519 109 109 109 178 178 178 F F F
118182024-02-04T14:05:31.093Z 26 520-539 103 103 103 172 172 172 F F F
118192024-02-04T14:05:31.093Z 27 540-559 109 109 109 178 178 178 F F F
118202024-02-04T14:05:31.093Z 28 560-579 106 106 106 174 174 174 F F F
118212024-02-04T14:05:31.093Z 29 580-599 106 106 106 175 175 175 F F F
118222024-02-04T14:05:31.093ZMax gen: 109, Max flush: 178
118232024-02-04T14:05:31.096Z
118242024-02-04T14:05:31.096ZBegin loop 55
118252024-02-04T14:05:31.097ZDownstairs to restart: 1
118262024-02-04T14:05:31.125Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.158228309Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569}
118272024-02-04T14:05:31.125Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.158697729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569}
118282024-02-04T14:05:31.129Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.158770022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569}
118292024-02-04T14:05:31.129Z{"msg":"Crucible 37c3876e-7fc9-4e4c-8b35-62ebeda27e32 has session id: 4fa966c8-14b3-41a1-95ef-c7af0eebb8a4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.158843113Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118302024-02-04T14:05:31.129Z{"msg":"Upstairs opts: Upstairs UUID: 37c3876e-7fc9-4e4c-8b35-62ebeda27e32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.158925061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118312024-02-04T14:05:31.129Z{"msg":"Crucible stats registered with UUID: 37c3876e-7fc9-4e4c-8b35-62ebeda27e32","v":0,"name":"crucible","level":30Crucible runtime is spawned
118322024-02-04T14:05:31.129Z,"time":"2024-02-04T14:05:30.159020952Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118332024-02-04T14:05:31.129Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159322881Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569}
118342024-02-04T14:05:31.129Z{"msg":"37c3876e-7fc9-4e4c-8b35-62ebeda27e32 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159538831Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118352024-02-04T14:05:31.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159591295Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118362024-02-04T14:05:31.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159621797Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118372024-02-04T14:05:31.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159691345Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118382024-02-04T14:05:31.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159735771Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118392024-02-04T14:05:31.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159807657Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118402024-02-04T14:05:31.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159852156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118412024-02-04T14:05:31.129Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159892348Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"io task","client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118422024-02-04T14:05:31.130Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.159969464Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"io task","client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118432024-02-04T14:05:31.130Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.160051432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"io task","client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118442024-02-04T14:05:31.130Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.160124473Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"io task","client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118452024-02-04T14:05:31.130Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.160207653Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"io task","client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118462024-02-04T14:05:31.130Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.16025187Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"io task","client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118472024-02-04T14:05:31.130Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.161433913Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118482024-02-04T14:05:31.130Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.161494834Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118492024-02-04T14:05:31.130Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.161533973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118502024-02-04T14:05:31.130Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.161607484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118512024-02-04T14:05:31.136Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.162739864Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118522024-02-04T14:05:31.136Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.162843565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118532024-02-04T14:05:31.136Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.163174476Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118542024-02-04T14:05:31.136Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.163253554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118552024-02-04T14:05:31.137Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.164050603Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118562024-02-04T14:05:31.137Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.164115259Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118572024-02-04T14:05:31.137Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.16414108Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118582024-02-04T14:05:31.137Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.164164524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118592024-02-04T14:05:31.137Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.164206092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118602024-02-04T14:05:31.137Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.164262078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118612024-02-04T14:05:31.137Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.164285826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118622024-02-04T14:05:31.137Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.164972957Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118632024-02-04T14:05:31.137Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165050461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118642024-02-04T14:05:31.137Z{"msg":"[0]R flush_numbers[0..12]: [139, 175, 172, 178, 178, 173, 175, 175, 178, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165081627Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118652024-02-04T14:05:31.137Z{"msg":"[0]R generation[0..12]: [87, 106, 103, 109, 109, 104, 106, 106, 109, 109, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165126032Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118662024-02-04T14:05:31.137Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165152162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118672024-02-04T14:05:31.137Z{"msg":"[1]R flush_numbers[0..12]: [139, 175, 172, 178, 178, 173, 175, 175, 178, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165195504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
118682024-02-04T14:05:31.137Z4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118692024-02-04T14:05:31.137Z{"msg":"[1]R generation[0..12]: [87, 106, 103, 109, 109, 104, 106, 106, 109, 109, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165223823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118702024-02-04T14:05:31.137Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165277723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118712024-02-04T14:05:31.137Z{"msg":"[2]R flush_numbers[0..12]: [139, 175, 172, 178, 178, 173, 175, 175, 178, 178, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165303236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118722024-02-04T14:05:31.137Z{"msg":"[2]R generation[0..12]: [87, 106, 103, 109, 109, 104, 106, 106, 109, 109, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165345265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118732024-02-04T14:05:31.137Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165371029Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118742024-02-04T14:05:31.137Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.16542082Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118752024-02-04T14:05:31.137Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165444829Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118762024-02-04T14:05:31.137Z{"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.16546676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118772024-02-04T14:05:31.137Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165507191Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118782024-02-04T14:05:31.138Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165531058Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118792024-02-04T14:05:31.138Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165574456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118802024-02-04T14:05:31.138Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165598112Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118812024-02-04T14:05:31.138Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165638961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118822024-02-04T14:05:31.138Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165662767Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118832024-02-04T14:05:31.138Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165703005Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118842024-02-04T14:05:31.138Z{"msg":"37c3876e-7fc9-4e4c-8b35-62ebeda27e32 is now active with session: 4fa966c8-14b3-41a1-95ef-c7af0eebb8a4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165726391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118852024-02-04T14:05:31.138Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165764623Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569,"session_id":"4fa966c8-14b3-41a1-95ef-c7af0eebb8a4"}
118862024-02-04T14:05:31.138Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:30.165786722Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569}
118872024-02-04T14:05:31.138ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118882024-02-04T14:05:31.141ZLoading write count information from file "/tmp/test_repair_verify.data"
118892024-02-04T14:05:31.141ZRun Repair workload
118902024-02-04T14:05:31.141Z01/30 Write block 366 len 24576 data: 9 11 10 10 11 9
118912024-02-04T14:05:31.141Z02/30 Write block 529 len 24576 data: 8 7 5 5 5 8
118922024-02-04T14:05:31.142Z03/30 Write block 351 len 32768 data: 10 10 11 9 10 11 8 7
118932024-02-04T14:05:31.142Z04/30 Write block 35 len 32768 data: 9 10 9 7 7 6 6 8
118942024-02-04T14:05:31.142Z05/30 Read block 203 len 20480
118952024-02-04T14:05:31.145Z06/30 Read block 73 len 24576
118962024-02-04T14:05:31.145Z07/30 Read block 470 len 24576
118972024-02-04T14:05:31.148Z08/30 Write block 44 len 8192 data: 9 9
118982024-02-04T14:05:31.148Z09/30 Read block 305 len 20480
118992024-02-04T14:05:31.151Z10/30 Write block 427 len 20480 data: 5 6 6 9 8
119002024-02-04T14:05:31.151Z11/30 Read block 289 len 40960
119012024-02-04T14:05:31.153Z12/30 Flush
119022024-02-04T14:05:31.157Z13/30 Write block 184 len 16384 data: 13 14 12 12
119032024-02-04T14:05:31.157Z14/30 Read block 550 len 24576
119042024-02-04T14:05:31.160Z15/30 Write block 355 len 24576 data: 11 12 9 8 7 7
119052024-02-04T14:05:31.160Z16/30 Read block 26 len 40960
119062024-02-04T14:05:31.163Z17/30 Read block 483 len 4096
119072024-02-04T14:05:31.163Z18/30 Read block 227 len 16384
119082024-02-04T14:05:31.163Z19/30 Write block 455 len 16384 data: 13 12 12 11
119092024-02-04T14:05:31.166Z20/30 Write block 355 len 8192 data: 12 13
119102024-02-04T14:05:31.166Z21/30 Write block 121 len 40960 data: 9 8 8 7 10 9 8 7 5 5
119112024-02-04T14:05:31.166Z22/30 Read block 364 len 16384
119122024-02-04T14:05:31.170Z23/30 Write block 260 len 28672 data: 9 11 12 11 10 10 11
119132024-02-04T14:05:31.172Z24/30 Read block 166 len 16384
119142024-02-04T14:05:31.173Z25/30 Write block 17 len 8192 data: 6 8
119152024-02-04T14:05:31.173Z26/30 Flush
119162024-02-04T14:05:31.177Z27/30 Write block 570 len 16384 data: 8 7 7 7
119172024-02-04T14:05:31.180Z28/30 Read block 338 len 8192
119182024-02-04T14:05:31.180Z29/30 Write block 434 len 28672 data: 11 11 11 9 8 6 9
119192024-02-04T14:05:31.184Z30/30 Write block 298 len 16384 data: 7 8 6 6
119202024-02-04T14:05:31.184Z----------------------------------------------------------------
119212024-02-04T14:05:31.184Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
119222024-02-04T14:05:31.184ZStates: Active Active Active
119232024-02-04T14:05:31.184ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119242024-02-04T14:05:31.184Z 1 Acked 1000 Write 6 Done Done Done false
119252024-02-04T14:05:31.184Z 2 Acked 1001 Write 6 Done Done Done false
119262024-02-04T14:05:31.184Z 3 Acked 1002 Write 8 Done Sent Done false
119272024-02-04T14:05:31.184Z 4 Acked 1003 Write 8 Done Sent Done false
119282024-02-04T14:05:31.184Z 5 Acked 1004 Read 5 Done Sent Done false
119292024-02-04T14:05:31.184Z 6 Acked 1005 Read 6 Done Sent Done false
119302024-02-04T14:05:31.184Z 7 Acked 1006 Read 6 Done Sent Done false
119312024-02-04T14:05:31.184Z 8 Acked 1007 Write 2 Done Sent Done false
119322024-02-04T14:05:31.184Z 9 Acked 1008 Read 5 Done Sent Done false
119332024-02-04T14:05:31.184Z 10 Acked 1009 Write 5 Done Sent Done false
119342024-02-04T14:05:31.184Z 11 Acked 1010 Read 10 Done Sent Done false
119352024-02-04T14:05:31.184Z 12 Acked 1011 Flush 0 Done Sent Done false
119362024-02-04T14:05:31.185Z 13 Acked 1012 Write 4 Done Sent Done false
119372024-02-04T14:05:31.185Z 14 Acked 1013 Read 6 Done Sent Done false
119382024-02-04T14:05:31.185Z 15 Acked 1014 Write 6 Done Sent Done false
119392024-02-04T14:05:31.185Z 16 Acked 1015 Read 10 Done Sent Done false
119402024-02-04T14:05:31.185Z 17 Acked 1016 Read 1 Done Sent Done false
119412024-02-04T14:05:31.185Z 18 Acked 1017 Read 4 Done Sent Done false
119422024-02-04T14:05:31.185Z 19 Acked 1018 Write 4 Done Sent Done false
119432024-02-04T14:05:31.185Z 20 Acked 1019 Write 2 Done Sent Done false
119442024-02-04T14:05:31.185Z 21 Acked 1020 Write 10 Done Sent Done false
119452024-02-04T14:05:31.185Z 22 Acked 1021 Read 4 Done Sent Done false
119462024-02-04T14:05:31.185Z 23 Acked 1022 Write 7 Done Sent Done false
119472024-02-04T14:05:31.185Z 24 Acked 1023 Read 4 Done Sent Done false
119482024-02-04T14:05:31.185Z 25 Acked 1024 Write 2 Done Sent Done false
119492024-02-04T14:05:31.185Z 26 Acked 1025 Flush 0 Done Sent Done false
119502024-02-04T14:05:31.185Z 27 Acked 1026 Write 4 Done Sent Done false
119512024-02-04T14:05:31.185Z 28 Acked 1027 Read 2 Done Sent Done false
119522024-02-04T14:05:31.185Z 29 Acked 1028 Write 7 Sent Sent Sent false
119532024-02-04T14:05:31.185Z 30 Acked 1029 Write 4 Sent Sent Sent false
119542024-02-04T14:05:31.185Z STATES DS:0 DS:1 DS:2 TOTAL
119552024-02-04T14:05:31.185Z New 0 0 0 0
119562024-02-04T14:05:31.185Z Sent 2 28 2 32
119572024-02-04T14:05:31.185Z Done 28 2 28 58
119582024-02-04T14:05:31.185Z Skipped 0 0 0 0
119592024-02-04T14:05:31.185Z Error 0 0 0 0
119602024-02-04T14:05:31.185ZLast Flush: 1025 0 1025
119612024-02-04T14:05:31.186ZDownstairs last five completed:
119622024-02-04T14:05:31.186ZUpstairs last five completed: 30 29 28 27 26
119632024-02-04T14:05:31.186Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:30.21519275Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1569}
119642024-02-04T14:05:31.186ZWrote out file "/tmp/test_repair_verify.data"
119652024-02-04T14:05:31.189Z
119662024-02-04T14:05:31.198ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119672024-02-04T14:05:31.221ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119682024-02-04T14:05:31.221Z 0 000-019 110 87 110 180 139 180 F F F <---
119692024-02-04T14:05:31.221Z 1 020-039 110 106 110 179 175 179 F F F <---
119702024-02-04T14:05:31.221Z 2 040-059 110 103 110 179 172 179 F F F <---
119712024-02-04T14:05:31.221Z 3 060-079 109 109 109 178 178 178 F F F
119722024-02-04T14:05:31.221Z 4 080-099 109 109 109 178 178 178 F F F
119732024-02-04T14:05:31.224Z 5 100-119 104 104 104 173 173 173 F F F
119742024-02-04T14:05:31.224Z 6 120-139 110 106 110 180 175 180 F F F <---
119752024-02-04T14:05:31.224Z 7 140-159 106 106 106 175 175 175 F F F
119762024-02-04T14:05:31.224Z 8 160-179 109 109 109 178 178 178 F F F
119772024-02-04T14:05:31.224Z 9 180-199 110 109 110 180 178 180 F F F <---
119782024-02-04T14:05:31.224Z 10 200-219 109 109 109 178 178 178 F F F
119792024-02-04T14:05:31.224Z 11 220-239 109 109 109 178 178 178 F F F
119802024-02-04T14:05:31.224Z 12 240-259 109 109 109 178 178 178 F F F
119812024-02-04T14:05:31.224Z 13 260-279 110 109 110 180 178 180 F F F <---
119822024-02-04T14:05:31.225Z 14 280-299 106 106 106 175 175 175 T F T <---
119832024-02-04T14:05:31.225Z 15 300-319 109 109 109 178 178 178 T F T <---
119842024-02-04T14:05:31.225Z 16 320-339 109 109 109 178 178 178 F F F
119852024-02-04T14:05:31.225Z 17 340-359 110 106 110 180 175 180 F F F <---
119862024-02-04T14:05:31.225Z 18 360-379 110 106 110 180 174 180 F T F <---
119872024-02-04T14:05:31.225Z 19 380-399 97 97 97 158 158 158 F F F
119882024-02-04T14:05:31.225Z 20 400-419 107 107 107 176 176 176 F F F
119892024-02-04T14:05:31.225Z 21 420-439 110 104 110 179 173 179 T F T <---
119902024-02-04T14:05:31.225Z 22 440-459 110 104 110 180 173 180 T F T <---
119912024-02-04T14:05:31.225Z 23 460-479 85 85 85 135 135 135 F F F
119922024-02-04T14:05:31.225Z 24 480-499 109 109 109 178 178 178 F F F
119932024-02-04T14:05:31.225Z 25 500-519 109 109 109 178 178 178 F F F
119942024-02-04T14:05:31.225Z 26 520-539 110 103 110 179 172 179 F T F <---
119952024-02-04T14:05:31.225Z 27 540-559 109 109 109 178 178 178 F F F
119962024-02-04T14:05:31.225Z 28 560-579 106 106 106 174 174 174 T F T <---
119972024-02-04T14:05:31.225Z 29 580-599 106 106 106 175 175 175 F F F
119982024-02-04T14:05:31.225ZMax gen: 110, Max flush: 180
119992024-02-04T14:05:31.225ZError: Difference in extent metadata found!
120002024-02-04T14:05:31.228ZA Difference in extent metadata is expected here
120012024-02-04T14:05:31.228ZOn loop 55
120022024-02-04T14:05:31.228Z
120032024-02-04T14:05:31.231ZVerifying data now
120042024-02-04T14:05:37.443ZNo change after verify
120052024-02-04T14:05:37.443ZLoop: 55 Downstairs dump after verify (and repair):
120062024-02-04T14:05:37.466ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120072024-02-04T14:05:37.466Z 0 000-019 110 110 110 180 180 180 F F F
120082024-02-04T14:05:37.466Z 1 020-039 110 110 110 179 179 179 F F F
120092024-02-04T14:05:37.466Z 2 040-059 110 110 110 179 179 179 F F F
120102024-02-04T14:05:37.466Z 3 060-079 109 109 109 178 178 178 F F F
120112024-02-04T14:05:37.469Z 4 080-099 109 109 109 178 178 178 F F F
120122024-02-04T14:05:37.470Z 5 100-119 104 104 104 173 173 173 F F F
120132024-02-04T14:05:37.470Z 6 120-139 110 110 110 180 180 180 F F F
120142024-02-04T14:05:37.470Z 7 140-159 106 106 106 175 175 175 F F F
120152024-02-04T14:05:37.470Z 8 160-179 109 109 109 178 178 178 F F F
120162024-02-04T14:05:37.470Z 9 180-199 110 110 110 180 180 180 F F F
120172024-02-04T14:05:37.470Z 10 200-219 109 109 109 178 178 178 F F F
120182024-02-04T14:05:37.470Z 11 220-239 109 109 109 178 178 178 F F F
120192024-02-04T14:05:37.470Z 12 240-259 109 109 109 178 178 178 F F F
120202024-02-04T14:05:37.470Z 13 260-279 110 110 110 180 180 180 F F F
120212024-02-04T14:05:37.470Z 14 280-299 111 111 111 181 181 181 F F F
120222024-02-04T14:05:37.470Z 15 300-319 111 111 111 181 181 181 F F F
120232024-02-04T14:05:37.470Z 16 320-339 109 109 109 178 178 178 F F F
120242024-02-04T14:05:37.470Z 17 340-359 110 110 110 180 180 180 F F F
120252024-02-04T14:05:37.470Z 18 360-379 110 110 110 180 180 180 F F F
120262024-02-04T14:05:37.470Z 19 380-399 97 97 97 158 158 158 F F F
120272024-02-04T14:05:37.470Z 20 400-419 107 107 107 176 176 176 F F F
120282024-02-04T14:05:37.470Z 21 420-439 111 111 111 181 181 181 F F F
120292024-02-04T14:05:37.470Z 22 440-459 111 111 111 181 181 181 F F F
120302024-02-04T14:05:37.470Z 23 460-479 85 85 85 135 135 135 F F F
120312024-02-04T14:05:37.470Z 24 480-499 109 109 109 178 178 178 F F F
120322024-02-04T14:05:37.470Z 25 500-519 109 109 109 178 178 178 F F F
120332024-02-04T14:05:37.470Z 26 520-539 110 110 110 179 179 179 F F F
120342024-02-04T14:05:37.470Z 27 540-559 109 109 109 178 178 178 F F F
120352024-02-04T14:05:37.471Z 28 560-579 111 111 111 181 181 181 F F F
120362024-02-04T14:05:37.471Z 29 580-599 106 106 106 175 175 175 F F F
120372024-02-04T14:05:37.471ZMax gen: 111, Max flush: 181
120382024-02-04T14:05:37.477Z
120392024-02-04T14:05:37.477ZBegin loop 56
120402024-02-04T14:05:37.477ZDownstairs to restart: 1
120412024-02-04T14:05:37.500Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.53276356Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577}
120422024-02-04T14:05:37.504Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533257443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577}
120432024-02-04T14:05:37.504Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
120442024-02-04T14:05:37.504Z,"time":"2024-02-04T14:05:36.533409708Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577}
120452024-02-04T14:05:37.504Z{"msg":"Crucible 0ece9de4-f5f7-48a3-992c-72a56dba70d3 has session id: a2ab9fe1-b766-4a1f-894a-8f2227823589","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533598895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120462024-02-04T14:05:37.504Z{"msg":"Upstairs opts: Upstairs UUID: 0ece9de4-f5f7-48a3-992c-72a56dba70d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533696602Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120472024-02-04T14:05:37.504Z{"msg":"Crucible stats registered with UUID: 0ece9de4-f5f7-48a3-992c-72a56dba70d3","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533748073Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120482024-02-04T14:05:37.504Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533771773Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577}
120492024-02-04T14:05:37.504Z{"msg":"0ece9de4-f5f7-48a3-992c-72a56dba70d3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533790986Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120502024-02-04T14:05:37.504Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533811631Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120512024-02-04T14:05:37.505Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533836258Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120522024-02-04T14:05:37.505Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.533964852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120532024-02-04T14:05:37.505Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534178322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120542024-02-04T14:05:37.505Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534361596Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120552024-02-04T14:05:37.505Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534460573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120562024-02-04T14:05:37.505Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534520907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120572024-02-04T14:05:37.505Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534600402Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120582024-02-04T14:05:37.505Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534648585Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120592024-02-04T14:05:37.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534690863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120602024-02-04T14:05:37.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534730943Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120612024-02-04T14:05:37.505Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.534772111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120622024-02-04T14:05:37.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.535500672Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120632024-02-04T14:05:37.505Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.535557347Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120642024-02-04T14:05:37.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.535686288Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120652024-02-04T14:05:37.505Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.535762418Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120662024-02-04T14:05:37.505Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.536433027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120672024-02-04T14:05:37.505Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.53661839Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120682024-02-04T14:05:37.509Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.536830485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120692024-02-04T14:05:37.510Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.536933201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120702024-02-04T14:05:37.510Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.537231336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120712024-02-04T14:05:37.510Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.537293098Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120722024-02-04T14:05:37.510Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.537319294Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120732024-02-04T14:05:37.510Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.537858757Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120742024-02-04T14:05:37.510Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.537932954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120752024-02-04T14:05:37.510Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.537958461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120762024-02-04T14:05:37.510Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.538090418Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120772024-02-04T14:05:37.510Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.538781423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120782024-02-04T14:05:37.510Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.53886292Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120792024-02-04T14:05:37.510Z{"msg":"[0]R flush_numbers[0..12]: [180, 179, 179, 178, 178, 173, 180, 175, 178, 180, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.538953552Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120802024-02-04T14:05:37.510Z{"msg":"[0]R generation[0..12]: [110, 110, 110, 109, 109, 104, 110, 106, 109, 110, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.53901836Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120812024-02-04T14:05:37.510Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539046201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120822024-02-04T14:05:37.510Z{"msg":"[1]R flush_numbers[0..12]: [180, 179, 179, 178, 178, 173, 180, 175, 178, 180, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539094113Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120832024-02-04T14:05:37.510Z{"msg":"[1]R generation[0..12]: [110, 110, 110, 109, 109, 104, 110, 106, 109, 110, 109, 109]","v":0,Wait for a query_work_queue command to finish before sending IO
120842024-02-04T14:05:37.510Z"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539123655Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120852024-02-04T14:05:37.510Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539173894Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120862024-02-04T14:05:37.510Z{"msg":"[2]R flush_numbers[0..12]: [180, 179, 179, 178, 178, 173, 180, 175, 178, 180, 178, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539198833Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120872024-02-04T14:05:37.510Z{"msg":"[2]R generation[0..12]: [110, 110, 110, 109, 109, 104, 110, 106, 109, 110, 109, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539247954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120882024-02-04T14:05:37.510Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539273001Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120892024-02-04T14:05:37.510Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539295182Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120902024-02-04T14:05:37.510Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539340291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120912024-02-04T14:05:37.510Z{"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539363273Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120922024-02-04T14:05:37.510Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539384861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120932024-02-04T14:05:37.511Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539424768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120942024-02-04T14:05:37.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539446249Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120952024-02-04T14:05:37.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539468541Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120962024-02-04T14:05:37.511Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539512039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120972024-02-04T14:05:37.511Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.53953536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120982024-02-04T14:05:37.511Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539581823Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
120992024-02-04T14:05:37.511Z{"msg":"0ece9de4-f5f7-48a3-992c-72a56dba70d3 is now active with session: a2ab9fe1-b766-4a1f-894a-8f2227823589","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539605065Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
121002024-02-04T14:05:37.511Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539625269Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577,"session_id":"a2ab9fe1-b766-4a1f-894a-8f2227823589"}
121012024-02-04T14:05:37.511Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:36.539645798Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577}
121022024-02-04T14:05:37.511ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
121032024-02-04T14:05:37.511ZLoading write count information from file "/tmp/test_repair_verify.data"
121042024-02-04T14:05:37.511ZRun Repair workload
121052024-02-04T14:05:37.511Z01/30 Write block 525 len 36864 data: 7 9 8 10 9 8 6 6 6
121062024-02-04T14:05:37.515Z02/30 Write block 75 len 40960 data: 9 9 11 10 9 10 11 12 10 10
121072024-02-04T14:05:37.516Z03/30 Read block 104 len 40960
121082024-02-04T14:05:37.518Z04/30 Read block 10 len 20480
121092024-02-04T14:05:37.518Z05/30 Write block 56 len 32768 data: 7 8 6 8 10 9 10 10
121102024-02-04T14:05:37.518Z06/30 Read block 385 len 12288
121112024-02-04T14:05:37.523Z07/30 Read block 334 len 16384
121122024-02-04T14:05:37.523Z08/30 Write block 108 len 36864 data: 11 10 10 11 14 12 13 13 11
121132024-02-04T14:05:37.523Z09/30 Read block 62 len 28672
121142024-02-04T14:05:37.526Z10/30 Write block 22 len 28672 data: 8 10 9 8 7 6 7
121152024-02-04T14:05:37.526Z11/30 Read block 460 len 32768
121162024-02-04T14:05:37.529Z12/30 Read block 377 len 36864
121172024-02-04T14:05:37.531Z13/30 Flush
121182024-02-04T14:05:37.534Z14/30 Write block 150 len 12288 data: 8 8 6
121192024-02-04T14:05:37.534Z15/30 Flush
121202024-02-04T14:05:37.537Z16/30 Write block 282 len 36864 data: 7 10 15 13 11 12 12 11 11
121212024-02-04T14:05:37.539Z17/30 Write block 417 len 16384 data: 9 8 9 12
121222024-02-04T14:05:37.540Z18/30 Write block 193 len 28672 data: 7 5 5 6 9 11 12
121232024-02-04T14:05:37.543Z19/30 Write block 26 len 20480 data: 8 7 8 7 8
121242024-02-04T14:05:37.543Z20/30 Write block 149 len 24576 data: 8 9 9 7 6 5
121252024-02-04T14:05:37.546Z21/30 Write block 173 len 24576 data: 12 11 10 9 12 11
121262024-02-04T14:05:37.546Z22/30 Write block 236 len 24576 data: 8 10 10 9 10 9
121272024-02-04T14:05:37.546Z23/30 Flush
121282024-02-04T14:05:37.550Z24/30 Write block 557 len 16384 data: 11 9 11 12
121292024-02-04T14:05:37.553Z25/30 Read block 260 len 12288
121302024-02-04T14:05:37.553Z26/30 Write block 126 len 20480 data: 10 9 8 6 6
121312024-02-04T14:05:37.558Z27/30 Write block 132 len 20480 data: 6 5 6 7 5
121322024-02-04T14:05:37.558Z28/30 Write block 56 len 20480 data: 8 9 7 9 11
121332024-02-04T14:05:37.562Z29/30 Write block 519 len 36864 data: 8 4 6 7 7 8 8 10 9
121342024-02-04T14:05:37.562Z30/30 Read block 132 len 36864
121352024-02-04T14:05:37.562Z----------------------------------------------------------------
121362024-02-04T14:05:37.562Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
121372024-02-04T14:05:37.562ZStates: Active Active Active
121382024-02-04T14:05:37.562ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121392024-02-04T14:05:37.562Z 1 Acked 1000 Write 9 Done Done Done false
121402024-02-04T14:05:37.562Z 2 Acked 1001 Write 10 Done Done Done false
121412024-02-04T14:05:37.562Z 3 Acked 1002 Read 10 Done Done Done false
121422024-02-04T14:05:37.562Z 4 Acked 1003 Read 5 Done Done Done false
121432024-02-04T14:05:37.562Z 5 Acked 1004 Write 8 Done Done Done false
121442024-02-04T14:05:37.562Z 6 Acked 1005 Read 3 Done Sent Done false
121452024-02-04T14:05:37.562Z 7 Acked 1006 Read 4 Done Sent Done false
121462024-02-04T14:05:37.562Z 8 Acked 1007 Write 9 Done Sent Done false
121472024-02-04T14:05:37.562Z 9 Acked 1008 Read 7 Done Sent Done false
121482024-02-04T14:05:37.562Z 10 Acked 1009 Write 7 Done Sent Done false
121492024-02-04T14:05:37.562Z 11 Acked 1010 Read 8 Done Sent Done false
121502024-02-04T14:05:37.562Z 12 Acked 1011 Read 9 Done Sent Done false
121512024-02-04T14:05:37.562Z 13 Acked 1012 Flush 0 Done Sent Done false
121522024-02-04T14:05:37.562Z 14 Acked 1013 Write 3 Done Sent Done false
121532024-02-04T14:05:37.563Z 15 Acked 1014 Flush 0 Done Sent Done false
121542024-02-04T14:05:37.563Z 16 Acked 1015 Write 9 Done Sent Done false
121552024-02-04T14:05:37.563Z 17 Acked 1016 Write 4 Done Sent Done false
121562024-02-04T14:05:37.563Z 18 Acked 1017 Write 7 Done Sent Done false
121572024-02-04T14:05:37.563Z 19 Acked 1018 Write 5 Done Sent Done false
121582024-02-04T14:05:37.563Z 20 Acked 1019 Write 6 Done Sent Done false
121592024-02-04T14:05:37.563Z 21 Acked 1020 Write 6 Done Sent Done false
121602024-02-04T14:05:37.563Z 22 Acked 1021 Write 6 Done Sent Done false
121612024-02-04T14:05:37.563Z 23 Acked 1022 Flush 0 Done Sent Done false
121622024-02-04T14:05:37.563Z 24 Acked 1023 Write 4 Done Sent Done false
121632024-02-04T14:05:37.563Z 25 Acked 1024 Read 3 Done Sent Done false
121642024-02-04T14:05:37.563Z 26 Acked 1025 Write 5 Done Sent Done false
121652024-02-04T14:05:37.563Z 27 Acked 1026 Write 5 Done Sent Done false
121662024-02-04T14:05:37.563Z 28 Acked 1027 Write 5 Done Sent Done false
121672024-02-04T14:05:37.563Z 29 Acked 1028 Write 9 Done Sent Done false
121682024-02-04T14:05:37.563Z 30 Acked 1029 Read 9 Done Sent Done false
121692024-02-04T14:05:37.563Z STATES DS:0 DS:1 DS:2 TOTAL
121702024-02-04T14:05:37.563Z New 0 0 0 0
121712024-02-04T14:05:37.563Z Sent 0 25 0 25
121722024-02-04T14:05:37.563Z Done 30 5 30 65
121732024-02-04T14:05:37.563Z Skipped 0 0 0 0
121742024-02-04T14:05:37.563Z Error 0 0 0 0
121752024-02-04T14:05:37.563ZLast Flush: 1022 0 1022
121762024-02-04T14:05:37.563ZDownstairs last five completed:
121772024-02-04T14:05:37.564ZUpstairs last five completed: 30 29 28 27 26
121782024-02-04T14:05:37.564Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:36.593488766Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1577}
121792024-02-04T14:05:37.564ZWrote out file "/tmp/test_repair_verify.data"
121802024-02-04T14:05:37.567Z
121812024-02-04T14:05:37.573ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121822024-02-04T14:05:37.600ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121832024-02-04T14:05:37.600Z 0 000-019 110 110 110 180 180 180 F F F
121842024-02-04T14:05:37.600Z 1 020-039 112 110 112 184 179 184 F F F <---
121852024-02-04T14:05:37.600Z 2 040-059 112 110 112 182 179 182 T T T <---
121862024-02-04T14:05:37.600Z 3 060-079 112 109 112 182 178 182 T T T <---
121872024-02-04T14:05:37.604Z 4 080-099 112 109 112 182 178 182 F T F <---
121882024-02-04T14:05:37.604Z 5 100-119 112 104 112 182 173 182 F F F <---
121892024-02-04T14:05:37.604Z 6 120-139 110 110 110 180 180 180 T F T <---
121902024-02-04T14:05:37.604Z 7 140-159 112 106 112 184 175 184 F F F <---
121912024-02-04T14:05:37.604Z 8 160-179 112 109 112 184 178 184 F F F <---
121922024-02-04T14:05:37.604Z 9 180-199 112 110 112 184 180 184 F F F <---
121932024-02-04T14:05:37.604Z 10 200-219 109 109 109 178 178 178 F F F
121942024-02-04T14:05:37.604Z 11 220-239 112 109 112 184 178 184 F F F <---
121952024-02-04T14:05:37.604Z 12 240-259 112 109 112 184 178 184 F F F <---
121962024-02-04T14:05:37.604Z 13 260-279 110 110 110 180 180 180 F F F
121972024-02-04T14:05:37.604Z 14 280-299 112 111 112 184 181 184 F F F <---
121982024-02-04T14:05:37.604Z 15 300-319 111 111 111 181 181 181 F F F
121992024-02-04T14:05:37.604Z 16 320-339 109 109 109 178 178 178 F F F
122002024-02-04T14:05:37.604Z 17 340-359 110 110 110 180 180 180 F F F
122012024-02-04T14:05:37.604Z 18 360-379 110 110 110 180 180 180 F F F
122022024-02-04T14:05:37.604Z 19 380-399 97 97 97 158 158 158 F F F
122032024-02-04T14:05:37.604Z 20 400-419 112 107 112 184 176 184 F F F <---
122042024-02-04T14:05:37.605Z 21 420-439 112 111 112 184 181 184 F F F <---
122052024-02-04T14:05:37.605Z 22 440-459 111 111 111 181 181 181 F F F
122062024-02-04T14:05:37.605Z 23 460-479 85 85 85 135 135 135 F F F
122072024-02-04T14:05:37.605Z 24 480-499 109 109 109 178 178 178 F F F
122082024-02-04T14:05:37.605Z 25 500-519 109 109 109 178 178 178 T F T <---
122092024-02-04T14:05:37.605Z 26 520-539 112 110 112 182 179 182 T T T <---
122102024-02-04T14:05:37.605Z 27 540-559 109 109 109 178 178 178 T F T <---
122112024-02-04T14:05:37.605Z 28 560-579 111 111 111 181 181 181 T F T <---
122122024-02-04T14:05:37.605Z 29 580-599 106 106 106 175 175 175 F F F
122132024-02-04T14:05:37.605ZMax gen: 112, Max flush: 184
122142024-02-04T14:05:37.605ZError: Difference in extent metadata found!
122152024-02-04T14:05:37.609ZA Difference in extent metadata is expected here
122162024-02-04T14:05:37.609ZOn loop 56
122172024-02-04T14:05:37.609Z
122182024-02-04T14:05:37.609ZVerifying data now
122192024-02-04T14:05:44.110ZNo change after verify
122202024-02-04T14:05:44.110ZLoop: 56 Downstairs dump after verify (and repair):
122212024-02-04T14:05:44.133ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122222024-02-04T14:05:44.133Z 0 000-019 110 110 110 180 180 180 F F F
122232024-02-04T14:05:44.133Z 1 020-039 112 112 112 184 184 184 F F F
122242024-02-04T14:05:44.133Z 2 040-059 113 113 113 185 185 185 F F F
122252024-02-04T14:05:44.133Z 3 060-079 113 113 113 185 185 185 F F F
122262024-02-04T14:05:44.134Z 4 080-099 112 112 112 182 182 182 F F F
122272024-02-04T14:05:44.137Z 5 100-119 112 112 112 182 182 182 F F F
122282024-02-04T14:05:44.137Z 6 120-139 113 113 113 185 185 185 F F F
122292024-02-04T14:05:44.137Z 7 140-159 112 112 112 184 184 184 F F F
122302024-02-04T14:05:44.137Z 8 160-179 112 112 112 184 184 184 F F F
122312024-02-04T14:05:44.137Z 9 180-199 112 112 112 184 184 184 F F F
122322024-02-04T14:05:44.137Z 10 200-219 109 109 109 178 178 178 F F F
122332024-02-04T14:05:44.137Z 11 220-239 112 112 112 184 184 184 F F F
122342024-02-04T14:05:44.138Z 12 240-259 112 112 112 184 184 184 F F F
122352024-02-04T14:05:44.138Z 13 260-279 110 110 110 180 180 180 F F F
122362024-02-04T14:05:44.138Z 14 280-299 112 112 112 184 184 184 F F F
122372024-02-04T14:05:44.138Z 15 300-319 111 111 111 181 181 181 F F F
122382024-02-04T14:05:44.138Z 16 320-339 109 109 109 178 178 178 F F F
122392024-02-04T14:05:44.138Z 17 340-359 110 110 110 180 180 180 F F F
122402024-02-04T14:05:44.138Z 18 360-379 110 110 110 180 180 180 F F F
122412024-02-04T14:05:44.138Z 19 380-399 97 97 97 158 158 158 F F F
122422024-02-04T14:05:44.138Z 20 400-419 112 112 112 184 184 184 F F F
122432024-02-04T14:05:44.138Z 21 420-439 112 112 112 184 184 184 F F F
122442024-02-04T14:05:44.138Z 22 440-459 111 111 111 181 181 181 F F F
122452024-02-04T14:05:44.138Z 23 460-479 85 85 85 135 135 135 F F F
122462024-02-04T14:05:44.139Z 24 480-499 109 109 109 178 178 178 F F F
122472024-02-04T14:05:44.139Z 25 500-519 113 113 113 185 185 185 F F F
122482024-02-04T14:05:44.139Z 26 520-539 113 113 113 185 185 185 F F F
122492024-02-04T14:05:44.139Z 27 540-559 113 113 113 185 185 185 F F F
122502024-02-04T14:05:44.139Z 28 560-579 113 113 113 185 185 185 F F F
122512024-02-04T14:05:44.139Z 29 580-599 106 106 106 175 175 175 F F F
122522024-02-04T14:05:44.139ZMax gen: 113, Max flush: 185
122532024-02-04T14:05:44.142Z
122542024-02-04T14:05:44.142ZBegin loop 57
122552024-02-04T14:05:44.143ZDownstairs to restart: 0
122562024-02-04T14:05:44.170Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.203048834Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585}
122572024-02-04T14:05:44.170Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.203539402Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585}
122582024-02-04T14:05:44.175Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.203614517Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585}
122592024-02-04T14:05:44.175Z{"msg":"Crucible 40cab72f-8759-49bc-ade9-8777602b0b99 has session id: a52fb626-32b0-4ae1-9d5f-76518ba52222","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.203656334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122602024-02-04T14:05:44.175Z{"msg":"Upstairs opts: Upstairs UUID: 40cab72f-8759-49bc-ade9-8777602b0b99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.203708597Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122612024-02-04T14:05:44.175Z{"msg":"Crucible runtime is spawned
122622024-02-04T14:05:44.175ZCrucible stats registered with UUID: 40cab72f-8759-49bc-ade9-8777602b0b99","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.203912135Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122632024-02-04T14:05:44.175Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204040992Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585}
122642024-02-04T14:05:44.175Z{"msg":"40cab72f-8759-49bc-ade9-8777602b0b99 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204134211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122652024-02-04T14:05:44.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204211577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122662024-02-04T14:05:44.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204281635Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122672024-02-04T14:05:44.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204324962Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122682024-02-04T14:05:44.176Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204401412Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122692024-02-04T14:05:44.176Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204441772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122702024-02-04T14:05:44.176Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204488507Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122712024-02-04T14:05:44.176Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204528162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122722024-02-04T14:05:44.176Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204654103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122732024-02-04T14:05:44.176Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204751809Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122742024-02-04T14:05:44.176Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204839197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122752024-02-04T14:05:44.176Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.204923342Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122762024-02-04T14:05:44.176Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.20495777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122772024-02-04T14:05:44.176Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.206156002Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122782024-02-04T14:05:44.176Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.206199381Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122792024-02-04T14:05:44.176Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.206332308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122802024-02-04T14:05:44.177Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.206359225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122812024-02-04T14:05:44.177Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.207054821Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122822024-02-04T14:05:44.183Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.207365137Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122832024-02-04T14:05:44.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.207892987Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122842024-02-04T14:05:44.183Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.208018542Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122852024-02-04T14:05:44.183Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.208125268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122862024-02-04T14:05:44.183Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.208165881Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122872024-02-04T14:05:44.184Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.208228005Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122882024-02-04T14:05:44.184Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.20825453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122892024-02-04T14:05:44.184Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.208310842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122902024-02-04T14:05:44.184Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.208335815Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122912024-02-04T14:05:44.184Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.20898768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122922024-02-04T14:05:44.184Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.209688359Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122932024-02-04T14:05:44.184Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.209760048Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122942024-02-04T14:05:44.184Z{"msg":"[0]R flush_numbers[0..12]: [180, 184, 185, 185, 182, 182, 185, 184, 184, 184, 178, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.209807049Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122952024-02-04T14:05:44.184Z{"msg":"[0]R generation[0..12]: [110, 112, 113, 113, 112, 112, 113, 112, 112, 112, 109, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.209862656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122962024-02-04T14:05:44.184Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.209908524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
122972024-02-04T14:05:44.184Z{"msg":"[1]R flush_numbers[0..12]: [180, 184, 185, 185, 182, 182, 185, 184, 184, 184, 178, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.209968711Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
122982024-02-04T14:05:44.184Za52fb626-32b0-4ae1-9d5f-76518ba52222"}
122992024-02-04T14:05:44.185Z{"msg":"[1]R generation[0..12]: [110, 112, 113, 113, 112, 112, 113, 112, 112, 112, 109, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.21001801Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123002024-02-04T14:05:44.185Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210063058Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123012024-02-04T14:05:44.185Z{"msg":"[2]R flush_numbers[0..12]: [180, 184, 185, 185, 182, 182, 185, 184, 184, 184, 178, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210121123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123022024-02-04T14:05:44.185Z{"msg":"[2]R generation[0..12]: [110, 112, 113, 113, 112, 112, 113, 112, 112, 112, 109, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210165828Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123032024-02-04T14:05:44.185Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210233714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123042024-02-04T14:05:44.185Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210290441Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123052024-02-04T14:05:44.185Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210333768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123062024-02-04T14:05:44.185Z{"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210372234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123072024-02-04T14:05:44.185Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210427037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123082024-02-04T14:05:44.185Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210469277Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123092024-02-04T14:05:44.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210524019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123102024-02-04T14:05:44.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210565816Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123112024-02-04T14:05:44.185Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210622315Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123122024-02-04T14:05:44.185Za52fb626-32b0-4ae1-9d5f-76518ba52222"}
123132024-02-04T14:05:44.186Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.21069343Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123142024-02-04T14:05:44.186Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210753331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123152024-02-04T14:05:44.186Z{"msg":"40cab72f-8759-49bc-ade9-8777602b0b99 is now active with session: a52fb626-32b0-4ae1-9d5f-76518ba52222","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210796777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123162024-02-04T14:05:44.186Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210853169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585,"session_id":"a52fb626-32b0-4ae1-9d5f-76518ba52222"}
123172024-02-04T14:05:44.186Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:43.210893042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585}
123182024-02-04T14:05:44.186ZLoading write count information from file "/tmp/test_repair_verify.data"
123192024-02-04T14:05:44.186ZRun Repair workload
123202024-02-04T14:05:44.186Z01/30 Write block 33 len 24576 data: 8 7 10 11 10 8
123212024-02-04T14:05:44.190Z02/30 Read block 530 len 12288
123222024-02-04T14:05:44.190Z03/30 Read block 298 len 8192
123232024-02-04T14:05:44.190Z04/30 Read block 533 len 12288
123242024-02-04T14:05:44.190Z05/30 Read block 442 len 24576
123252024-02-04T14:05:44.190Z06/30 Flush
123262024-02-04T14:05:44.190Z07/30 Write block 548 len 12288 data: 6 7 6
123272024-02-04T14:05:44.193Z08/30 Write block 370 len 20480 data: 12 10 6 6 9
123282024-02-04T14:05:44.193Z09/30 Read block 130 len 12288
123292024-02-04T14:05:44.193Z10/30 Read block 234 len 32768
123302024-02-04T14:05:44.196Z11/30 Write block 223 len 12288 data: 6 6 7
123312024-02-04T14:05:44.196Z12/30 Read block 164 len 20480
123322024-02-04T14:05:44.219Z13/30 Read block 152 len 28672
123332024-02-04T14:05:44.219Z14/30 Write block 308 len 28672 data: 7 8 9 9 7 8 7
123342024-02-04T14:05:44.223Z15/30 Read block 590 len 24576
123352024-02-04T14:05:44.223Z16/30 Write block 431 len 20480 data: 9 9 9 12 12
123362024-02-04T14:05:44.223Z17/30 Write block 395 len 4096 data: 12
123372024-02-04T14:05:44.223Z18/30 Write block 375 len 40960 data: 9 7 6 8 8 11 9 6 6 7
123382024-02-04T14:05:44.223Z19/30 Write block 240 len 36864 data: 11 10 7 6 6 6 8 9 13
123392024-02-04T14:05:44.223Z20/30 Read block 13 len 40960
123402024-02-04T14:05:44.223Z21/30 Write block 228 len 20480 data: 9 8 7 6 6
123412024-02-04T14:05:44.223Z22/30 Write block 220 len 28672 data: 9 10 7 7 7 8 7
123422024-02-04T14:05:44.223Z23/30 Read block 317 len 20480
123432024-02-04T14:05:44.223Z24/30 Read block 63 len 12288
123442024-02-04T14:05:44.223Z25/30 Read block 118 len 32768
123452024-02-04T14:05:44.223Z26/30 Write block 522 len 12288 data: 8 8 9
123462024-02-04T14:05:44.224Z27/30 Write block 158 len 12288 data: 4 6 6
123472024-02-04T14:05:44.224Z28/30 Write block 244 len 32768 data: 7 7 9 10 14 14 15 14
123482024-02-04T14:05:44.224Z29/30 Write block 152 len 28672 data: 8 7 6 5 4 5 5
123492024-02-04T14:05:44.224Z30/30 Read block 47 len 12288
123502024-02-04T14:05:44.228Z----------------------------------------------------------------
123512024-02-04T14:05:44.228Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123522024-02-04T14:05:44.228ZStates: Active Active Active
123532024-02-04T14:05:44.228ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123542024-02-04T14:05:44.228Z 1 Acked 1000 Write 6 Done Done Done false
123552024-02-04T14:05:44.228Z 2 Acked 1001 Read 3 Done Done Done false
123562024-02-04T14:05:44.228Z 3 Acked 1002 Read 2 Sent Done Done false
123572024-02-04T14:05:44.228Z 4 Acked 1003 Read 3 Sent Done Done false
123582024-02-04T14:05:44.229Z 5 Acked 1004 Read 6 Sent Done Done false
123592024-02-04T14:05:44.229Z 6 Acked 1005 Flush 0 Sent Done Done false
123602024-02-04T14:05:44.229Z 7 Acked 1006 Write 3 Sent Done Done false
123612024-02-04T14:05:44.229Z 8 Acked 1007 Write 5 Sent Done Done false
123622024-02-04T14:05:44.229Z 9 Acked 1008 Read 3 Sent Done Done false
123632024-02-04T14:05:44.229Z 10 Acked 1009 Read 8 Sent Done Done false
123642024-02-04T14:05:44.229Z 11 Acked 1010 Write 3 Sent Done Done false
123652024-02-04T14:05:44.229Z 12 Acked 1011 Read 5 Sent Done Done false
123662024-02-04T14:05:44.229Z 13 Acked 1012 Read 7 Sent Done Done false
123672024-02-04T14:05:44.229Z 14 Acked 1013 Write 7 Sent Done Done false
123682024-02-04T14:05:44.229Z 15 Acked 1014 Read 6 Sent Done Done false
123692024-02-04T14:05:44.229Z 16 Acked 1015 Write 5 Sent Done Done false
123702024-02-04T14:05:44.229Z 17 Acked 1016 Write 1 Sent Done Done false
123712024-02-04T14:05:44.229Z 18 Acked 1017 Write 10 Sent Done Done false
123722024-02-04T14:05:44.229Z 19 Acked 1018 Write 9 Sent Done Done false
123732024-02-04T14:05:44.229Z 20 Acked 1019 Read 10 Sent Done Done false
123742024-02-04T14:05:44.229Z 21 Acked 1020 Write 5 Sent Done Done false
123752024-02-04T14:05:44.229Z 22 Acked 1021 Write 7 Sent Done Done false
123762024-02-04T14:05:44.229Z 23 Acked 1022 Read 5 Sent Done Done false
123772024-02-04T14:05:44.229Z 24 Acked 1023 Read 3 Sent Done Done false
123782024-02-04T14:05:44.229Z 25 Acked 1024 Read 8 Sent Done Done false
123792024-02-04T14:05:44.229Z 26 Acked 1025 Write 3 Sent Done Done false
123802024-02-04T14:05:44.229Z 27 Acked 1026 Write 3 Sent Done Done false
123812024-02-04T14:05:44.229Z 28 Acked 1027 Write 8 Sent Done Done false
123822024-02-04T14:05:44.230Z 29 Acked 1028 Write 7 Sent Done Sent false
123832024-02-04T14:05:44.230Z 30 Acked 1029 Read 3 Sent Done Sent false
123842024-02-04T14:05:44.230Z STATES DS:0 DS:1 DS:2 TOTAL
123852024-02-04T14:05:44.230Z New 0 0 0 0
123862024-02-04T14:05:44.230Z Sent 28 0 2 30
123872024-02-04T14:05:44.230Z Done 2 30 28 60
123882024-02-04T14:05:44.230Z Skipped 0 0 0 0
123892024-02-04T14:05:44.230Z Error 0 0 0 0
123902024-02-04T14:05:44.230ZLast Flush: 0 1005 1005
123912024-02-04T14:05:44.230ZDownstairs last five completed:
123922024-02-04T14:05:44.230ZUpstairs last five completed: 30 29 28 27 26
123932024-02-04T14:05:44.230Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:43.256638618Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1585}
123942024-02-04T14:05:44.230ZWrote out file "/tmp/test_repair_verify.data"
123952024-02-04T14:05:44.234Z
123962024-02-04T14:05:44.237ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123972024-02-04T14:05:44.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123982024-02-04T14:05:44.267Z 0 000-019 110 110 110 180 180 180 F F F
123992024-02-04T14:05:44.267Z 1 020-039 112 114 114 184 186 186 T F F <---
124002024-02-04T14:05:44.267Z 2 040-059 113 113 113 185 185 185 F F F
124012024-02-04T14:05:44.267Z 3 060-079 113 113 113 185 185 185 F F F
124022024-02-04T14:05:44.267Z 4 080-099 112 112 112 182 182 182 F F F
124032024-02-04T14:05:44.267Z 5 100-119 112 112 112 182 182 182 F F F
124042024-02-04T14:05:44.267Z 6 120-139 113 113 113 185 185 185 F F F
124052024-02-04T14:05:44.267Z 7 140-159 112 112 112 184 184 184 F T T <---
124062024-02-04T14:05:44.267Z 8 160-179 112 112 112 184 184 184 F T T <---
124072024-02-04T14:05:44.267Z 9 180-199 112 112 112 184 184 184 F F F
124082024-02-04T14:05:44.268Z 10 200-219 109 109 109 178 178 178 F F F
124092024-02-04T14:05:44.268Z 11 220-239 112 112 112 184 184 184 F T T <---
124102024-02-04T14:05:44.268Z 12 240-259 112 112 112 184 184 184 F T T <---
124112024-02-04T14:05:44.268Z 13 260-279 110 110 110 180 180 180 F F F
124122024-02-04T14:05:44.268Z 14 280-299 112 112 112 184 184 184 F F F
124132024-02-04T14:05:44.271Z 15 300-319 111 111 111 181 181 181 F T T <---
124142024-02-04T14:05:44.271Z 16 320-339 109 109 109 178 178 178 F F F
124152024-02-04T14:05:44.271Z 17 340-359 110 110 110 180 180 180 F F F
124162024-02-04T14:05:44.271Z 18 360-379 110 110 110 180 180 180 F T T <---
124172024-02-04T14:05:44.272Z 19 380-399 97 97 97 158 158 158 F T T <---
124182024-02-04T14:05:44.272Z 20 400-419 112 112 112 184 184 184 F F F
124192024-02-04T14:05:44.272Z 21 420-439 112 112 112 184 184 184 F T T <---
124202024-02-04T14:05:44.272Z 22 440-459 111 111 111 181 181 181 F F F
124212024-02-04T14:05:44.272Z 23 460-479 85 85 85 135 135 135 F F F
124222024-02-04T14:05:44.272Z 24 480-499 109 109 109 178 178 178 F F F
124232024-02-04T14:05:44.272Z 25 500-519 113 113 113 185 185 185 F F F
124242024-02-04T14:05:44.272Z 26 520-539 113 113 113 185 185 185 F T T <---
124252024-02-04T14:05:44.272Z 27 540-559 113 113 113 185 185 185 F T T <---
124262024-02-04T14:05:44.272Z 28 560-579 113 113 113 185 185 185 F F F
124272024-02-04T14:05:44.272Z 29 580-599 106 106 106 175 175 175 F F F
124282024-02-04T14:05:44.272ZMax gen: 114, Max flush: 186
124292024-02-04T14:05:44.272ZError: Difference in extent metadata found!
124302024-02-04T14:05:44.276ZA Difference in extent metadata is expected here
124312024-02-04T14:05:44.276ZOn loop 57
124322024-02-04T14:05:44.276Z
124332024-02-04T14:05:44.280ZVerifying data now
124342024-02-04T14:05:50.281ZNo change after verify
124352024-02-04T14:05:50.281ZLoop: 57 Downstairs dump after verify (and repair):
124362024-02-04T14:05:50.307ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124372024-02-04T14:05:50.307Z 0 000-019 110 110 110 180 180 180 F F F
124382024-02-04T14:05:50.307Z 1 020-039 114 114 114 186 186 186 F F F
124392024-02-04T14:05:50.307Z 2 040-059 113 113 113 185 185 185 F F F
124402024-02-04T14:05:50.307Z 3 060-079 113 113 113 185 185 185 F F F
124412024-02-04T14:05:50.307Z 4 080-099 112 112 112 182 182 182 F F F
124422024-02-04T14:05:50.307Z 5 100-119 112 112 112 182 182 182 F F F
124432024-02-04T14:05:50.307Z 6 120-139 113 113 113 185 185 185 F F F
124442024-02-04T14:05:50.307Z 7 140-159 115 115 115 187 187 187 F F F
124452024-02-04T14:05:50.311Z 8 160-179 115 115 115 187 187 187 F F F
124462024-02-04T14:05:50.311Z 9 180-199 112 112 112 184 184 184 F F F
124472024-02-04T14:05:50.311Z 10 200-219 109 109 109 178 178 178 F F F
124482024-02-04T14:05:50.311Z 11 220-239 115 115 115 187 187 187 F F F
124492024-02-04T14:05:50.311Z 12 240-259 115 115 115 187 187 187 F F F
124502024-02-04T14:05:50.311Z 13 260-279 110 110 110 180 180 180 F F F
124512024-02-04T14:05:50.311Z 14 280-299 112 112 112 184 184 184 F F F
124522024-02-04T14:05:50.311Z 15 300-319 115 115 115 187 187 187 F F F
124532024-02-04T14:05:50.311Z 16 320-339 109 109 109 178 178 178 F F F
124542024-02-04T14:05:50.311Z 17 340-359 110 110 110 180 180 180 F F F
124552024-02-04T14:05:50.311Z 18 360-379 115 115 115 187 187 187 F F F
124562024-02-04T14:05:50.311Z 19 380-399 115 115 115 187 187 187 F F F
124572024-02-04T14:05:50.311Z 20 400-419 112 112 112 184 184 184 F F F
124582024-02-04T14:05:50.312Z 21 420-439 115 115 115 187 187 187 F F F
124592024-02-04T14:05:50.312Z 22 440-459 111 111 111 181 181 181 F F F
124602024-02-04T14:05:50.312Z 23 460-479 85 85 85 135 135 135 F F F
124612024-02-04T14:05:50.312Z 24 480-499 109 109 109 178 178 178 F F F
124622024-02-04T14:05:50.312Z 25 500-519 113 113 113 185 185 185 F F F
124632024-02-04T14:05:50.312Z 26 520-539 115 115 115 187 187 187 F F F
124642024-02-04T14:05:50.312Z 27 540-559 115 115 115 187 187 187 F F F
124652024-02-04T14:05:50.312Z 28 560-579 113 113 113 185 185 185 F F F
124662024-02-04T14:05:50.312Z 29 580-599 106 106 106 175 175 175 F F F
124672024-02-04T14:05:50.312ZMax gen: 115, Max flush: 187
124682024-02-04T14:05:50.312Z
124692024-02-04T14:05:50.312ZBegin loop 58
124702024-02-04T14:05:50.312ZDownstairs to restart: 1
124712024-02-04T14:05:50.339Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.372039769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593}
124722024-02-04T14:05:50.339Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.372486266Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593}
124732024-02-04T14:05:50.343Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.372557155Z","hostname":"Crucible runtime is spawned
124742024-02-04T14:05:50.343Zip-10-150-1-226.us-west-2.compute.internal","pid":1593}
124752024-02-04T14:05:50.343Z{"msg":"Crucible eefeaaf7-990c-4740-a359-e6e89e24449b has session id: 76dcfea9-977a-4888-b7b0-8c426a3485a5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.372702953Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124762024-02-04T14:05:50.343Z{"msg":"Upstairs opts: Upstairs UUID: eefeaaf7-990c-4740-a359-e6e89e24449b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.372799424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124772024-02-04T14:05:50.343Z{"msg":"Crucible stats registered with UUID: eefeaaf7-990c-4740-a359-e6e89e24449b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.372875482Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124782024-02-04T14:05:50.343Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.372942973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593}
124792024-02-04T14:05:50.343Z{"msg":"eefeaaf7-990c-4740-a359-e6e89e24449b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.372989186Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124802024-02-04T14:05:50.343Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373062652Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124812024-02-04T14:05:50.343Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373104606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124822024-02-04T14:05:50.343Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373201805Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124832024-02-04T14:05:50.343Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373246042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124842024-02-04T14:05:50.343Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373284228Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124852024-02-04T14:05:50.343Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373324123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124862024-02-04T14:05:50.344Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373361019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"io task","client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124872024-02-04T14:05:50.344Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373400159Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"io task","client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124882024-02-04T14:05:50.344Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373438561Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"io task","client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124892024-02-04T14:05:50.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373573172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"io task","client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124902024-02-04T14:05:50.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373880968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"io task","client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124912024-02-04T14:05:50.344Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.373989806Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"io task","client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124922024-02-04T14:05:50.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.374920741Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124932024-02-04T14:05:50.344Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.375006445Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124942024-02-04T14:05:50.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.375077914Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124952024-02-04T14:05:50.344Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.375124091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124962024-02-04T14:05:50.348Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.375906112Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124972024-02-04T14:05:50.348Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.376227573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124982024-02-04T14:05:50.348Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.376572748Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
124992024-02-04T14:05:50.348Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.376666398Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125002024-02-04T14:05:50.349Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.376754328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125012024-02-04T14:05:50.349Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.376836566Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125022024-02-04T14:05:50.349Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.376884449Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125032024-02-04T14:05:50.349Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.377008234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125042024-02-04T14:05:50.349Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.377066994Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125052024-02-04T14:05:50.349Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.377092579Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125062024-02-04T14:05:50.349Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.377389672Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125072024-02-04T14:05:50.349Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378085058Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125082024-02-04T14:05:50.349Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.37815646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125092024-02-04T14:05:50.349Z{"msg":"[0]R flush_numbers[0..12]: [180, 186, 185, 185, 182, 182, 185, 187, 187, 184, 178, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378193774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125102024-02-04T14:05:50.349Z{"msg":"[0]R generation[0..12]: [110, 114, 113, 113, 112, 112, 113, 115, 115, 112, 109, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378218802Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125112024-02-04T14:05:50.349Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.37826338Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125122024-02-04T14:05:50.349Z{"msg":"[1]R flush_numbers[0..12]: [180, 186, 185, 185, 182, 182, 185, 187, 187, 184, 178, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378290067Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125132024-02-04T14:05:50.349Z{"msg":"[1]R generation[0..12]: [110, 114, 113, 113, 112, 112, 113, 115, 115, 112, 109, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.37833217Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125142024-02-04T14:05:50.349Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.3783577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125152024-02-04T14:05:50.349ZWait for a query_work_queue command to finish before sending IO
125162024-02-04T14:05:50.349Z{"msg":"[2]R flush_numbers[0..12]: [180, 186, 185, 185, 182, 182, 185, 187, 187, 184, 178, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.37840538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125172024-02-04T14:05:50.349Z{"msg":"[2]R generation[0..12]: [110, 114, 113, 113, 112, 112, 113, 115, 115, 112, 109, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378430269Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125182024-02-04T14:05:50.349Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378472659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125192024-02-04T14:05:50.349Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378496073Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125202024-02-04T14:05:50.349Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378550472Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125212024-02-04T14:05:50.349Z{"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378573331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125222024-02-04T14:05:50.349Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378595353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125232024-02-04T14:05:50.350Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378640445Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125242024-02-04T14:05:50.350Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378662266Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125252024-02-04T14:05:50.350Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378684691Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125262024-02-04T14:05:50.350Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378738807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125272024-02-04T14:05:50.350Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378762336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125282024-02-04T14:05:50.350Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378783209Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125292024-02-04T14:05:50.350Z{"msg":"eefeaaf7-990c-4740-a359-e6e89e24449b is now active with session: 76dcfea9-977a-4888-b7b0-8c426a3485a5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378830746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125302024-02-04T14:05:50.350Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378852141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593,"session_id":"76dcfea9-977a-4888-b7b0-8c426a3485a5"}
125312024-02-04T14:05:50.350Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:49.378873006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593}
125322024-02-04T14:05:50.350ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125332024-02-04T14:05:50.350ZLoading write count information from file "/tmp/test_repair_verify.data"
125342024-02-04T14:05:50.350ZRun Repair workload
125352024-02-04T14:05:50.350Z01/30 Write block 492 len 8192 data: 15 14
125362024-02-04T14:05:50.353Z02/30 Read block 409 len 4096
125372024-02-04T14:05:50.353Z03/30 Read block 336 len 12288
125382024-02-04T14:05:50.353Z04/30 Write block 564 len 20480 data: 11 12 12 10 9
125392024-02-04T14:05:50.353Z05/30 Read block 521 len 20480
125402024-02-04T14:05:50.356Z06/30 Write block 374 len 8192 data: 10 10
125412024-02-04T14:05:50.356Z07/30 Write block 539 len 36864 data: 6 7 7 6 9 7 7 5 5
125422024-02-04T14:05:50.359Z08/30 Flush
125432024-02-04T14:05:50.362Z09/30 Write block 332 len 36864 data: 6 6 7 6 6 7 7 7 8
125442024-02-04T14:05:50.362Z10/30 Read block 261 len 40960
125452024-02-04T14:05:50.365Z11/30 Read block 179 len 40960
125462024-02-04T14:05:50.368Z12/30 Read block 238 len 20480
125472024-02-04T14:05:50.370Z13/30 Read block 77 len 24576
125482024-02-04T14:05:50.370Z14/30 Read block 317 len 12288
125492024-02-04T14:05:50.373Z15/30 Write block 263 len 8192 data: 12 11
125502024-02-04T14:05:50.373Z16/30 Read block 312 len 12288
125512024-02-04T14:05:50.376Z17/30 Write block 188 len 36864 data: 10 10 7 8 8 8 6 6 7
125522024-02-04T14:05:50.376Z18/30 Read block 34 len 40960
125532024-02-04T14:05:50.379Z19/30 Read block 499 len 32768
125542024-02-04T14:05:50.379Z20/30 Write block 151 len 24576 data: 10 9 8 7 6 5
125552024-02-04T14:05:50.381Z21/30 Read block 363 len 28672
125562024-02-04T14:05:50.381Z22/30 Flush
125572024-02-04T14:05:50.388Z23/30 Write block 492 len 28672 data: 16 15 11 10 13 11 12
125582024-02-04T14:05:50.391Z24/30 Flush
125592024-02-04T14:05:50.391Z25/30 Write block 476 len 40960 data: 8 7 7 7 5 4 2 4 6 9
125602024-02-04T14:05:50.391Z26/30 Flush
125612024-02-04T14:05:50.395Z27/30 Write block 578 len 24576 data: 12 14 16 15 11 12
125622024-02-04T14:05:50.398Z28/30 Write block 404 len 28672 data: 9 12 11 11 10 11 11
125632024-02-04T14:05:50.398Z29/30 Read block 4 len 4096
125642024-02-04T14:05:50.401Z30/30 Write block 422 len 4096 data: 9
125652024-02-04T14:05:50.401Z----------------------------------------------------------------
125662024-02-04T14:05:50.401Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
125672024-02-04T14:05:50.401ZStates: Active Active Active
125682024-02-04T14:05:50.401ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125692024-02-04T14:05:50.401Z 1 Acked 1000 Write 2 Done Sent Done false
125702024-02-04T14:05:50.401Z 2 Acked 1001 Read 1 Done Sent Done false
125712024-02-04T14:05:50.401Z 3 Acked 1002 Read 3 Done Sent Done false
125722024-02-04T14:05:50.401Z 4 Acked 1003 Write 5 Done Sent Done false
125732024-02-04T14:05:50.401Z 5 Acked 1004 Read 5 Done Sent Done false
125742024-02-04T14:05:50.401Z 6 Acked 1005 Write 2 Done Sent Done false
125752024-02-04T14:05:50.401Z 7 Acked 1006 Write 9 Done Sent Done false
125762024-02-04T14:05:50.401Z 8 Acked 1007 Flush 0 Done Sent Done false
125772024-02-04T14:05:50.401Z 9 Acked 1008 Write 9 Done Sent Done false
125782024-02-04T14:05:50.401Z 10 Acked 1009 Read 10 Done Sent Done false
125792024-02-04T14:05:50.402Z 11 Acked 1010 Read 10 Done Sent Done false
125802024-02-04T14:05:50.402Z 12 Acked 1011 Read 5 Done Sent Done false
125812024-02-04T14:05:50.402Z 13 Acked 1012 Read 6 Done Sent Done false
125822024-02-04T14:05:50.402Z 14 Acked 1013 Read 3 Done Sent Done false
125832024-02-04T14:05:50.402Z 15 Acked 1014 Write 2 Done Sent Done false
125842024-02-04T14:05:50.402Z 16 Acked 1015 Read 3 Done Sent Done false
125852024-02-04T14:05:50.402Z 17 Acked 1016 Write 9 Done Sent Done false
125862024-02-04T14:05:50.402Z 18 Acked 1017 Read 10 Done Sent Done false
125872024-02-04T14:05:50.402Z 19 Acked 1018 Read 8 Done Sent Done false
125882024-02-04T14:05:50.402Z 20 Acked 1019 Write 6 Done Sent Done false
125892024-02-04T14:05:50.402Z 21 Acked 1020 Read 7 Done Sent Done false
125902024-02-04T14:05:50.402Z 22 Acked 1021 Flush 0 Done Sent Done false
125912024-02-04T14:05:50.402Z 23 Acked 1022 Write 7 Done Sent Done false
125922024-02-04T14:05:50.402Z 24 Acked 1023 Flush 0 Done Sent Done false
125932024-02-04T14:05:50.402Z 25 Acked 1024 Write 10 Done Sent Done false
125942024-02-04T14:05:50.402Z 26 Acked 1025 Flush 0 Done Sent Done false
125952024-02-04T14:05:50.402Z 27 Acked 1026 Write 6 Done Sent Done false
125962024-02-04T14:05:50.402Z 28 Acked 1027 Write 7 Done Sent Done false
125972024-02-04T14:05:50.402Z 29 Acked 1028 Read 1 Done Sent Done false
125982024-02-04T14:05:50.402Z 30 Acked 1029 Write 1 Sent Sent Sent false
125992024-02-04T14:05:50.402Z STATES DS:0 DS:1 DS:2 TOTAL
126002024-02-04T14:05:50.402Z New 0 0 0 0
126012024-02-04T14:05:50.402Z Sent 1 30 1 32
126022024-02-04T14:05:50.402Z Done 29 0 29 58
126032024-02-04T14:05:50.403Z Skipped 0 0 0 0
126042024-02-04T14:05:50.403Z Error 0 0 0 0
126052024-02-04T14:05:50.403ZLast Flush: 1025 0 1025
126062024-02-04T14:05:50.403ZDownstairs last five completed:
126072024-02-04T14:05:50.403ZUpstairs last five completed: 30 29 28 27 26
126082024-02-04T14:05:50.403Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:49.432882984Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1593}
126092024-02-04T14:05:50.403ZWrote out file "/tmp/test_repair_verify.data"
126102024-02-04T14:05:50.407Z
126112024-02-04T14:05:50.414ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126122024-02-04T14:05:50.439ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126132024-02-04T14:05:50.439Z 0 000-019 110 110 110 180 180 180 F F F
126142024-02-04T14:05:50.439Z 1 020-039 114 114 114 186 186 186 F F F
126152024-02-04T14:05:50.439Z 2 040-059 113 113 113 185 185 185 F F F
126162024-02-04T14:05:50.439Z 3 060-079 113 113 113 185 185 185 F F F
126172024-02-04T14:05:50.440Z 4 080-099 112 112 112 182 182 182 F F F
126182024-02-04T14:05:50.440Z 5 100-119 112 112 112 182 182 182 F F F
126192024-02-04T14:05:50.440Z 6 120-139 113 113 113 185 185 185 F F F
126202024-02-04T14:05:50.443Z 7 140-159 116 115 116 189 187 189 F F F <---
126212024-02-04T14:05:50.443Z 8 160-179 115 115 115 187 187 187 F F F
126222024-02-04T14:05:50.443Z 9 180-199 116 112 116 189 184 189 F F F <---
126232024-02-04T14:05:50.443Z 10 200-219 109 109 109 178 178 178 F F F
126242024-02-04T14:05:50.443Z 11 220-239 115 115 115 187 187 187 F F F
126252024-02-04T14:05:50.443Z 12 240-259 115 115 115 187 187 187 F F F
126262024-02-04T14:05:50.443Z 13 260-279 116 110 116 189 180 189 F F F <---
126272024-02-04T14:05:50.443Z 14 280-299 112 112 112 184 184 184 F F F
126282024-02-04T14:05:50.443Z 15 300-319 115 115 115 187 187 187 F F F
126292024-02-04T14:05:50.443Z 16 320-339 116 109 116 189 178 189 F F F <---
126302024-02-04T14:05:50.443Z 17 340-359 116 110 116 189 180 189 F F F <---
126312024-02-04T14:05:50.443Z 18 360-379 116 115 116 188 187 188 F F F <---
126322024-02-04T14:05:50.443Z 19 380-399 115 115 115 187 187 187 F F F
126332024-02-04T14:05:50.443Z 20 400-419 112 112 112 184 184 184 T F T <---
126342024-02-04T14:05:50.444Z 21 420-439 115 115 115 187 187 187 T F T <---
126352024-02-04T14:05:50.444Z 22 440-459 111 111 111 181 181 181 F F F
126362024-02-04T14:05:50.444Z 23 460-479 116 85 116 191 135 191 F F F <---
126372024-02-04T14:05:50.444Z 24 480-499 116 109 116 191 178 191 F F F <---
126382024-02-04T14:05:50.444Z 25 500-519 113 113 113 185 185 185 F F F
126392024-02-04T14:05:50.444Z 26 520-539 116 115 116 188 187 188 F F F <---
126402024-02-04T14:05:50.444Z 27 540-559 116 115 116 188 187 188 F F F <---
126412024-02-04T14:05:50.444Z 28 560-579 116 113 116 188 185 188 T F T <---
126422024-02-04T14:05:50.444Z 29 580-599 106 106 106 175 175 175 T F T <---
126432024-02-04T14:05:50.444ZMax gen: 116, Max flush: 191
126442024-02-04T14:05:50.444ZError: Difference in extent metadata found!
126452024-02-04T14:05:50.452ZA Difference in extent metadata is expected here
126462024-02-04T14:05:50.452ZOn loop 58
126472024-02-04T14:05:50.452Z
126482024-02-04T14:05:50.452ZVerifying data now
126492024-02-04T14:05:55.658ZNo change after verify
126502024-02-04T14:05:55.658ZLoop: 58 Downstairs dump after verify (and repair):
126512024-02-04T14:05:55.681ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126522024-02-04T14:05:55.681Z 0 000-019 110 110 110 180 180 180 F F F
126532024-02-04T14:05:55.681Z 1 020-039 114 114 114 186 186 186 F F F
126542024-02-04T14:05:55.681Z 2 040-059 113 113 113 185 185 185 F F F
126552024-02-04T14:05:55.681Z 3 060-079 113 113 113 185 185 185 F F F
126562024-02-04T14:05:55.681Z 4 080-099 112 112 112 182 182 182 F F F
126572024-02-04T14:05:55.681Z 5 100-119 112 112 112 182 182 182 F F F
126582024-02-04T14:05:55.681Z 6 120-139 113 113 113 185 185 185 F F F
126592024-02-04T14:05:55.684Z 7 140-159 116 116 116 189 189 189 F F F
126602024-02-04T14:05:55.684Z 8 160-179 115 115 115 187 187 187 F F F
126612024-02-04T14:05:55.684Z 9 180-199 116 116 116 189 189 189 F F F
126622024-02-04T14:05:55.684Z 10 200-219 109 109 109 178 178 178 F F F
126632024-02-04T14:05:55.685Z 11 220-239 115 115 115 187 187 187 F F F
126642024-02-04T14:05:55.685Z 12 240-259 115 115 115 187 187 187 F F F
126652024-02-04T14:05:55.685Z 13 260-279 116 116 116 189 189 189 F F F
126662024-02-04T14:05:55.685Z 14 280-299 112 112 112 184 184 184 F F F
126672024-02-04T14:05:55.685Z 15 300-319 115 115 115 187 187 187 F F F
126682024-02-04T14:05:55.685Z 16 320-339 116 116 116 189 189 189 F F F
126692024-02-04T14:05:55.685Z 17 340-359 116 116 116 189 189 189 F F F
126702024-02-04T14:05:55.685Z 18 360-379 116 116 116 188 188 188 F F F
126712024-02-04T14:05:55.685Z 19 380-399 115 115 115 187 187 187 F F F
126722024-02-04T14:05:55.685Z 20 400-419 117 117 117 192 192 192 F F F
126732024-02-04T14:05:55.685Z 21 420-439 117 117 117 192 192 192 F F F
126742024-02-04T14:05:55.685Z 22 440-459 111 111 111 181 181 181 F F F
126752024-02-04T14:05:55.685Z 23 460-479 116 116 116 191 191 191 F F F
126762024-02-04T14:05:55.685Z 24 480-499 116 116 116 191 191 191 F F F
126772024-02-04T14:05:55.685Z 25 500-519 113 113 113 185 185 185 F F F
126782024-02-04T14:05:55.685Z 26 520-539 116 116 116 188 188 188 F F F
126792024-02-04T14:05:55.685Z 27 540-559 116 116 116 188 188 188 F F F
126802024-02-04T14:05:55.685Z 28 560-579 117 117 117 192 192 192 F F F
126812024-02-04T14:05:55.685Z 29 580-599 117 117 117 192 192 192 F F F
126822024-02-04T14:05:55.685ZMax gen: 117, Max flush: 192
126832024-02-04T14:05:55.688Z
126842024-02-04T14:05:55.688ZBegin loop 59
126852024-02-04T14:05:55.688ZDownstairs to restart: 0
126862024-02-04T14:05:55.716Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.748435429Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601}
126872024-02-04T14:05:55.716Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749028053Z","hostname"Crucible runtime is spawned
126882024-02-04T14:05:55.716Z:"ip-10-150-1-226.us-west-2.compute.internal","pid":1601}
126892024-02-04T14:05:55.720Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.74911238Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601}
126902024-02-04T14:05:55.720Z{"msg":"Crucible 63e52b8c-c67d-48f1-98b5-97689e0f5a2f has session id: 45053e34-5188-4369-b73c-c083f79e87fa","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749184546Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
126912024-02-04T14:05:55.720Z{"msg":"Upstairs opts: Upstairs UUID: 63e52b8c-c67d-48f1-98b5-97689e0f5a2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749238408Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
126922024-02-04T14:05:55.720Z{"msg":"Crucible stats registered with UUID: 63e52b8c-c67d-48f1-98b5-97689e0f5a2f","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749294219Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
126932024-02-04T14:05:55.720Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749335153Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601}
126942024-02-04T14:05:55.720Z{"msg":"63e52b8c-c67d-48f1-98b5-97689e0f5a2f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749386626Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
126952024-02-04T14:05:55.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749426612Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
126962024-02-04T14:05:55.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749515031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
126972024-02-04T14:05:55.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749674298Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
126982024-02-04T14:05:55.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.74971761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
126992024-02-04T14:05:55.720Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749784602Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127002024-02-04T14:05:55.720Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749819286Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127012024-02-04T14:05:55.720Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.74985949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"io task","client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127022024-02-04T14:05:55.720Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749902645Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"io task","client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127032024-02-04T14:05:55.721Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.749948405Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"io task","client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127042024-02-04T14:05:55.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.750065967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"io task","client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127052024-02-04T14:05:55.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.750116666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"io task","client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127062024-02-04T14:05:55.721Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.750211108Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"io task","client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127072024-02-04T14:05:55.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.751277656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127082024-02-04T14:05:55.721Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.751515503Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127092024-02-04T14:05:55.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.751565192Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127102024-02-04T14:05:55.721Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.751608092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127112024-02-04T14:05:55.721Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.752184427Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127122024-02-04T14:05:55.721Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.752248634Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127132024-02-04T14:05:55.721Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.752400212Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127142024-02-04T14:05:55.721Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.752470394Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127152024-02-04T14:05:55.725Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.753207439Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127162024-02-04T14:05:55.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.753635929Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127172024-02-04T14:05:55.725Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.753747103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127182024-02-04T14:05:55.725Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.753813461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127192024-02-04T14:05:55.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.753839596Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127202024-02-04T14:05:55.725Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.753910946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127212024-02-04T14:05:55.725Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.753958092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127222024-02-04T14:05:55.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754160729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127232024-02-04T14:05:55.726Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754217541Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127242024-02-04T14:05:55.726Z{"msg":"[0]R flush_numbers[0..12]: [180, 186, 185, 185, 182, 182, 185, 189, 187, 189, 178, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754244224Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127252024-02-04T14:05:55.726Z{"msg":"[0]R generation[0..12]: [110, 114, 113, 113, 112, 112, 113, 116, 115, 116, 109, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754268312Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127262024-02-04T14:05:55.726Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754312042Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127272024-02-04T14:05:55.726Z{"msg":"[1]R flush_numbers[0..12]: [180, 186, 185, 185, 182, 182, 185, 189, 187, 189, 178, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754337294Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127282024-02-04T14:05:55.726Z{"msg":"[1]R generation[0..12]: [110, 114, 113, 113, 112, 112, 113, 116, 115, 116, 109, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754385758Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127292024-02-04T14:05:55.726Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754410976Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127302024-02-04T14:05:55.726Z{"msg":"[2]R flush_numbers[0..12]: [180, 186, 185, 185, 182, 182, 185, 189, 187, 189, 178, 187]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
127312024-02-04T14:05:55.726Z,"time":"2024-02-04T14:05:54.754467475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127322024-02-04T14:05:55.726Z{"msg":"[2]R generation[0..12]: [110, 114, 113, 113, 112, 112, 113, 116, 115, 116, 109, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.75449571Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127332024-02-04T14:05:55.726Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754563837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127342024-02-04T14:05:55.726Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754587733Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127352024-02-04T14:05:55.726Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754630644Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127362024-02-04T14:05:55.726Z{"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754653452Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127372024-02-04T14:05:55.726Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754675427Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127382024-02-04T14:05:55.726Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754715954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127392024-02-04T14:05:55.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.75473777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127402024-02-04T14:05:55.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754760299Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127412024-02-04T14:05:55.726Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754782238Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127422024-02-04T14:05:55.726Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754823092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127432024-02-04T14:05:55.726Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.75484507Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127442024-02-04T14:05:55.726Z{"msg":"63e52b8c-c67d-48f1-98b5-97689e0f5a2f is now active with session: 45053e34-5188-4369-b73c-c083f79e87fa","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754892488Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127452024-02-04T14:05:55.726Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754914127Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601,"session_id":"45053e34-5188-4369-b73c-c083f79e87fa"}
127462024-02-04T14:05:55.727Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:54.754953959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601}
127472024-02-04T14:05:55.727ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127482024-02-04T14:05:55.727ZLoading write count information from file "/tmp/test_repair_verify.data"
127492024-02-04T14:05:55.727ZRun Repair workload
127502024-02-04T14:05:55.727Z01/30 Write block 182 len 4096 data: 12
127512024-02-04T14:05:55.730Z02/30 Write block 5 len 20480 data: 7 7 7 7 9
127522024-02-04T14:05:55.730Z03/30 Read block 354 len 28672
127532024-02-04T14:05:55.730Z04/30 Read block 232 len 20480
127542024-02-04T14:05:55.730Z05/30 Flush
127552024-02-04T14:05:55.733Z06/30 Write block 564 len 8192 data: 12 13
127562024-02-04T14:05:55.733Z07/30 Read block 181 len 32768
127572024-02-04T14:05:55.736Z08/30 Write block 226 len 40960 data: 8 8 10 9 8 7 7 8 9 7
127582024-02-04T14:05:55.736Z09/30 Read block 200 len 12288
127592024-02-04T14:05:55.739Z10/30 Read block 63 len 16384
127602024-02-04T14:05:55.742Z11/30 Read block 504 len 24576
127612024-02-04T14:05:55.742Z12/30 Write block 348 len 36864 data: 10 12 9 11 11 12 10 13 14
127622024-02-04T14:05:55.742Z13/30 Read block 85 len 12288
127632024-02-04T14:05:55.748Z14/30 Read block 2 len 20480
127642024-02-04T14:05:55.751Z15/30 Read block 546 len 32768
127652024-02-04T14:05:55.751Z16/30 Write block 9 len 12288 data: 10 6 5
127662024-02-04T14:05:55.751Z17/30 Read block 104 len 28672
127672024-02-04T14:05:55.751Z18/30 Write block 551 len 40960 data: 4 4 8 8 8 10 12 10 12 13
127682024-02-04T14:05:55.754Z19/30 Write block 586 len 4096 data: 12
127692024-02-04T14:05:55.754Z20/30 Flush
127702024-02-04T14:05:55.757Z21/30 Write block 182 len 8192 data: 13 15
127712024-02-04T14:05:55.757Z22/30 Read block 287 len 4096
127722024-02-04T14:05:55.760Z23/30 Read block 492 len 12288
127732024-02-04T14:05:55.760Z24/30 Read block 406 len 8192
127742024-02-04T14:05:55.760Z25/30 Write block 589 len 24576 data: 10 9 7 8 9 10
127752024-02-04T14:05:55.763Z26/30 Read block 52 len 36864
127762024-02-04T14:05:55.763Z27/30 Write block 66 len 12288 data: 7 10 10
127772024-02-04T14:05:55.766Z28/30 Write block 34 len 20480 data: 8 11 12 11 9
127782024-02-04T14:05:55.766Z29/30 Read block 64 len 36864
127792024-02-04T14:05:55.770Z30/30 Write block 292 len 8192 data: 8 5
127802024-02-04T14:05:55.770Z----------------------------------------------------------------
127812024-02-04T14:05:55.770Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
127822024-02-04T14:05:55.770ZStates: Active Active Active
127832024-02-04T14:05:55.771ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127842024-02-04T14:05:55.771Z 1 Acked 1000 Write 1 Done Done Done false
127852024-02-04T14:05:55.771Z 2 Acked 1001 Write 5 Sent Done Done false
127862024-02-04T14:05:55.771Z 3 Acked 1002 Read 7 Sent Done Done false
127872024-02-04T14:05:55.771Z 4 Acked 1003 Read 5 Sent Done Done false
127882024-02-04T14:05:55.771Z 5 Acked 1004 Flush 0 Sent Done Done false
127892024-02-04T14:05:55.771Z 6 Acked 1005 Write 2 Sent Done Done false
127902024-02-04T14:05:55.771Z 7 Acked 1006 Read 8 Sent Done Done false
127912024-02-04T14:05:55.771Z 8 Acked 1007 Write 10 Sent Done Done false
127922024-02-04T14:05:55.771Z 9 Acked 1008 Read 3 Sent Done Done false
127932024-02-04T14:05:55.771Z 10 Acked 1009 Read 4 Sent Done Done false
127942024-02-04T14:05:55.771Z 11 Acked 1010 Read 6 Sent Done Done false
127952024-02-04T14:05:55.771Z 12 Acked 1011 Write 9 Sent Done Done false
127962024-02-04T14:05:55.771Z 13 Acked 1012 Read 3 Sent Done Done false
127972024-02-04T14:05:55.771Z 14 Acked 1013 Read 5 Sent Done Done false
127982024-02-04T14:05:55.771Z 15 Acked 1014 Read 8 Sent Done Done false
127992024-02-04T14:05:55.771Z 16 Acked 1015 Write 3 Sent Done Done false
128002024-02-04T14:05:55.771Z 17 Acked 1016 Read 7 Sent Done Done false
128012024-02-04T14:05:55.771Z 18 Acked 1017 Write 10 Sent Done Done false
128022024-02-04T14:05:55.771Z 19 Acked 1018 Write 1 Sent Done Done false
128032024-02-04T14:05:55.771Z 20 Acked 1019 Flush 0 Sent Done Done false
128042024-02-04T14:05:55.771Z 21 Acked 1020 Write 2 Sent Done Done false
128052024-02-04T14:05:55.772Z 22 Acked 1021 Read 1 Sent Done Done false
128062024-02-04T14:05:55.772Z 23 Acked 1022 Read 3 Sent Done Done false
128072024-02-04T14:05:55.772Z 24 Acked 1023 Read 2 Sent Done Done false
128082024-02-04T14:05:55.772Z 25 Acked 1024 Write 6 Sent Done Done false
128092024-02-04T14:05:55.772Z 26 Acked 1025 Read 9 Sent Done Done false
128102024-02-04T14:05:55.772Z 27 Acked 1026 Write 3 Sent Done Done false
128112024-02-04T14:05:55.772Z 28 Acked 1027 Write 5 Sent Done Done false
128122024-02-04T14:05:55.772Z 29 Acked 1028 Read 9 Sent Done Done false
128132024-02-04T14:05:55.772Z 30 Acked 1029 Write 2 Sent Sent Sent false
128142024-02-04T14:05:55.772Z STATES DS:0 DS:1 DS:2 TOTAL
128152024-02-04T14:05:55.772Z New 0 0 0 0
128162024-02-04T14:05:55.772Z Sent 29 1 1 31
128172024-02-04T14:05:55.772Z Done 1 29 29 59
128182024-02-04T14:05:55.772Z Skipped 0 0 0 0
128192024-02-04T14:05:55.772Z Error 0 0 0 0
128202024-02-04T14:05:55.772ZLast Flush: 0 1019 1019
128212024-02-04T14:05:55.772ZDownstairs last five completed:
128222024-02-04T14:05:55.772ZUpstairs last five completed: 30 29 28 27 26
128232024-02-04T14:05:55.772Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:54.801008302Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1601}
128242024-02-04T14:05:55.772ZWrote out file "/tmp/test_repair_verify.data"
128252024-02-04T14:05:55.776Z
128262024-02-04T14:05:55.781ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128272024-02-04T14:05:55.807ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128282024-02-04T14:05:55.807Z 0 000-019 110 118 118 180 194 194 F F F <---
128292024-02-04T14:05:55.807Z 1 020-039 114 114 114 186 186 186 F T T <---
128302024-02-04T14:05:55.807Z 2 040-059 113 113 113 185 185 185 F F F
128312024-02-04T14:05:55.807Z 3 060-079 113 113 113 185 185 185 F T T <---
128322024-02-04T14:05:55.807Z 4 080-099 112 112 112 182 182 182 F F F
128332024-02-04T14:05:55.810Z 5 100-119 112 112 112 182 182 182 F F F
128342024-02-04T14:05:55.810Z 6 120-139 113 113 113 185 185 185 F F F
128352024-02-04T14:05:55.810Z 7 140-159 116 116 116 189 189 189 F F F
128362024-02-04T14:05:55.810Z 8 160-179 115 115 115 187 187 187 F F F
128372024-02-04T14:05:55.810Z 9 180-199 116 118 118 189 193 193 T T T <---
128382024-02-04T14:05:55.810Z 10 200-219 109 109 109 178 178 178 F F F
128392024-02-04T14:05:55.810Z 11 220-239 115 118 118 187 194 194 F F F <---
128402024-02-04T14:05:55.810Z 12 240-259 115 115 115 187 187 187 F F F
128412024-02-04T14:05:55.811Z 13 260-279 116 116 116 189 189 189 F F F
128422024-02-04T14:05:55.811Z 14 280-299 112 112 112 184 184 184 F T T <---
128432024-02-04T14:05:55.811Z 15 300-319 115 115 115 187 187 187 F F F
128442024-02-04T14:05:55.811Z 16 320-339 116 116 116 189 189 189 F F F
128452024-02-04T14:05:55.811Z 17 340-359 116 118 118 189 194 194 F F F <---
128462024-02-04T14:05:55.811Z 18 360-379 116 116 116 188 188 188 F F F
128472024-02-04T14:05:55.811Z 19 380-399 115 115 115 187 187 187 F F F
128482024-02-04T14:05:55.811Z 20 400-419 117 117 117 192 192 192 F F F
128492024-02-04T14:05:55.811Z 21 420-439 117 117 117 192 192 192 F F F
128502024-02-04T14:05:55.811Z 22 440-459 111 111 111 181 181 181 F F F
128512024-02-04T14:05:55.811Z 23 460-479 116 116 116 191 191 191 F F F
128522024-02-04T14:05:55.811Z 24 480-499 116 116 116 191 191 191 F F F
128532024-02-04T14:05:55.811Z 25 500-519 113 113 113 185 185 185 F F F
128542024-02-04T14:05:55.811Z 26 520-539 116 116 116 188 188 188 F F F
128552024-02-04T14:05:55.811Z 27 540-559 116 118 118 188 194 194 F F F <---
128562024-02-04T14:05:55.811Z 28 560-579 117 118 118 192 194 194 F F F <---
128572024-02-04T14:05:55.811Z 29 580-599 117 118 118 192 194 194 F T T <---
128582024-02-04T14:05:55.811ZMax gen: 118, Max flush: 194
128592024-02-04T14:05:55.811ZError: Difference in extent metadata found!
128602024-02-04T14:05:55.814ZA Difference in extent metadata is expected here
128612024-02-04T14:05:55.814ZOn loop 59
128622024-02-04T14:05:55.814Z
128632024-02-04T14:05:55.817ZVerifying data now
128642024-02-04T14:06:00.727ZNo change after verify
128652024-02-04T14:06:00.728ZLoop: 59 Downstairs dump after verify (and repair):
128662024-02-04T14:06:00.751ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128672024-02-04T14:06:00.751Z 0 000-019 118 118 118 194 194 194 F F F
128682024-02-04T14:06:00.751Z 1 020-039 119 119 119 195 195 195 F F F
128692024-02-04T14:06:00.751Z 2 040-059 113 113 113 185 185 185 F F F
128702024-02-04T14:06:00.751Z 3 060-079 119 119 119 195 195 195 F F F
128712024-02-04T14:06:00.751Z 4 080-099 112 112 112 182 182 182 F F F
128722024-02-04T14:06:00.755Z 5 100-119 112 112 112 182 182 182 F F F
128732024-02-04T14:06:00.755Z 6 120-139 113 113 113 185 185 185 F F F
128742024-02-04T14:06:00.756Z 7 140-159 116 116 116 189 189 189 F F F
128752024-02-04T14:06:00.756Z 8 160-179 115 115 115 187 187 187 F F F
128762024-02-04T14:06:00.756Z 9 180-199 119 119 119 195 195 195 F F F
128772024-02-04T14:06:00.756Z 10 200-219 109 109 109 178 178 178 F F F
128782024-02-04T14:06:00.756Z 11 220-239 118 118 118 194 194 194 F F F
128792024-02-04T14:06:00.756Z 12 240-259 115 115 115 187 187 187 F F F
128802024-02-04T14:06:00.756Z 13 260-279 116 116 116 189 189 189 F F F
128812024-02-04T14:06:00.756Z 14 280-299 119 119 119 195 195 195 F F F
128822024-02-04T14:06:00.756Z 15 300-319 115 115 115 187 187 187 F F F
128832024-02-04T14:06:00.756Z 16 320-339 116 116 116 189 189 189 F F F
128842024-02-04T14:06:00.756Z 17 340-359 118 118 118 194 194 194 F F F
128852024-02-04T14:06:00.756Z 18 360-379 116 116 116 188 188 188 F F F
128862024-02-04T14:06:00.757Z 19 380-399 115 115 115 187 187 187 F F F
128872024-02-04T14:06:00.757Z 20 400-419 117 117 117 192 192 192 F F F
128882024-02-04T14:06:00.757Z 21 420-439 117 117 117 192 192 192 F F F
128892024-02-04T14:06:00.757Z 22 440-459 111 111 111 181 181 181 F F F
128902024-02-04T14:06:00.757Z 23 460-479 116 116 116 191 191 191 F F F
128912024-02-04T14:06:00.757Z 24 480-499 116 116 116 191 191 191 F F F
128922024-02-04T14:06:00.757Z 25 500-519 113 113 113 185 185 185 F F F
128932024-02-04T14:06:00.757Z 26 520-539 116 116 116 188 188 188 F F F
128942024-02-04T14:06:00.757Z 27 540-559 118 118 118 194 194 194 F F F
128952024-02-04T14:06:00.757Z 28 560-579 118 118 118 194 194 194 F F F
128962024-02-04T14:06:00.757Z 29 580-599 119 119 119 195 195 195 F F F
128972024-02-04T14:06:00.757ZMax gen: 119, Max flush: 195
128982024-02-04T14:06:00.758Z
128992024-02-04T14:06:00.758ZBegin loop 60
129002024-02-04T14:06:00.758ZDownstairs to restart: 1
129012024-02-04T14:06:00.785Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.817590678Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609}
129022024-02-04T14:06:00.785Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818026098Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609}
129032024-02-04T14:06:00.785Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818071191Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609}
129042024-02-04T14:06:00.801Z{"msg":"Crucible 2784a371-b826-4f59-b09b-d7490673d272 has session id: 7a254390-f607-448d-a5c2-5b658d029c52","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818131283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129052024-02-04T14:06:00.801Z{"msg":"Upstairs opts: Upstairs UUID: 2784a371-b826-4f59-b09b-d7490673d272, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
129062024-02-04T14:06:00.801Z,"time":"2024-02-04T14:05:59.81823795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129072024-02-04T14:06:00.801Z{"msg":"Crucible stats registered with UUID: 2784a371-b826-4f59-b09b-d7490673d272","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818285225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129082024-02-04T14:06:00.801Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818352099Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609}
129092024-02-04T14:06:00.801Z{"msg":"2784a371-b826-4f59-b09b-d7490673d272 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818671642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129102024-02-04T14:06:00.801Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818724665Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129112024-02-04T14:06:00.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818754643Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129122024-02-04T14:06:00.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818819276Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129132024-02-04T14:06:00.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818864613Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129142024-02-04T14:06:00.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818906981Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129152024-02-04T14:06:00.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.818982752Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129162024-02-04T14:06:00.802Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.819025106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"io task","client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129172024-02-04T14:06:00.802Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.819101983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"io task","client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129182024-02-04T14:06:00.802Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.819239913Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"io task","client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129192024-02-04T14:06:00.802Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.819312296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"io task","client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129202024-02-04T14:06:00.802Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.819341793Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"io task","client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129212024-02-04T14:06:00.802Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.819365084Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"io task","client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129222024-02-04T14:06:00.802Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.820750299Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129232024-02-04T14:06:00.803Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.820820593Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129242024-02-04T14:06:00.803Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.821014843Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129252024-02-04T14:06:00.803Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.821102322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129262024-02-04T14:06:00.803Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.821183053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129272024-02-04T14:06:00.803Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.821265488Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129282024-02-04T14:06:00.808Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.82184999Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129292024-02-04T14:06:00.808Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.822163323Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129302024-02-04T14:06:00.808Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.822238359Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129312024-02-04T14:06:00.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.822638424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129322024-02-04T14:06:00.808Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.822713322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129332024-02-04T14:06:00.808Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.822744122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129342024-02-04T14:06:00.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.82301807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129352024-02-04T14:06:00.809Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823060459Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129362024-02-04T14:06:00.809Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823084647Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129372024-02-04T14:06:00.809Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823151188Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129382024-02-04T14:06:00.809Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823197369Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129392024-02-04T14:06:00.809Z{"msg":"[0]R flush_numbers[0..12]: [194, 195, 185, 195, 182, 182, 185, 189, 187, 195, 178, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823237059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129402024-02-04T14:06:00.809Z{"msg":"[0]R generation[0..12]: [118, 119, 113, 119, 112, 112, 113, 116, 115, 119, 109, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823281483Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129412024-02-04T14:06:00.809Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823307537Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129422024-02-04T14:06:00.809Z{"msg":"[1]R flush_numbers[0..12]: [194, 195, 185, 195, 182, 182, 185, 189, 187, 195, 178, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823351267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129432024-02-04T14:06:00.809Z{"msg":"[1]R generation[0..12]: [118, 119, 113, 119, 112, 112, 113, 116, 115, 119, 109, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823376235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129442024-02-04T14:06:00.809Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823420405Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129452024-02-04T14:06:00.809Z{"msg":"[2]R flush_numbers[0..12]: [194, 195, 185, 195, 182, 182, 185, 189, 187, 195, 178, 194]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
129462024-02-04T14:06:00.809Z,"time":"2024-02-04T14:05:59.823445869Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129472024-02-04T14:06:00.810Z{"msg":"[2]R generation[0..12]: [118, 119, 113, 119, 112, 112, 113, 116, 115, 119, 109, 118]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823562761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129482024-02-04T14:06:00.810Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823647017Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129492024-02-04T14:06:00.810Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823700335Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129502024-02-04T14:06:00.810Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823724798Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129512024-02-04T14:06:00.810Z{"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823746599Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129522024-02-04T14:06:00.810Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823786927Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129532024-02-04T14:06:00.810Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823810519Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129542024-02-04T14:06:00.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823832375Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129552024-02-04T14:06:00.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823855051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129562024-02-04T14:06:00.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823877415Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129572024-02-04T14:06:00.810Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823899758Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129582024-02-04T14:06:00.810Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823921878Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129592024-02-04T14:06:00.810Z{"msg":"2784a371-b826-4f59-b09b-d7490673d272 is now active with session: 7a254390-f607-448d-a5c2-5b658d029c52","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823943983Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129602024-02-04T14:06:00.811Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.823964011Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609,"session_id":"7a254390-f607-448d-a5c2-5b658d029c52"}
129612024-02-04T14:06:00.811Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:05:59.824003364Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609}
129622024-02-04T14:06:00.811ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129632024-02-04T14:06:00.811ZLoading write count information from file "/tmp/test_repair_verify.data"
129642024-02-04T14:06:00.811ZRun Repair workload
129652024-02-04T14:06:00.811Z01/30 Write block 241 len 16384 data: 11 8 7 8
129662024-02-04T14:06:00.811Z02/30 Write block 114 len 24576 data: 14 14 12 12 9 9
129672024-02-04T14:06:00.811Z03/30 Write block 105 len 32768 data: 6 6 7 12 11 11 12 15
129682024-02-04T14:06:00.811Z04/30 Read block 213 len 4096
129692024-02-04T14:06:00.811Z05/30 Read block 295 len 20480
129702024-02-04T14:06:00.811Z06/30 Write block 223 len 4096 data: 8
129712024-02-04T14:06:00.811Z07/30 Write block 280 len 24576 data: 8 7 8 11 16 14
129722024-02-04T14:06:00.811Z08/30 Write block 493 len 12288 data: 16 12 11
129732024-02-04T14:06:00.812Z09/30 Read block 30 len 36864
129742024-02-04T14:06:00.812Z10/30 Flush
129752024-02-04T14:06:00.816Z11/30 Flush
129762024-02-04T14:06:00.816Z12/30 Flush
129772024-02-04T14:06:00.816Z13/30 Write block 65 len 24576 data: 9 8 11 11 8 6
129782024-02-04T14:06:00.816Z14/30 Read block 359 len 28672
129792024-02-04T14:06:00.816Z15/30 Write block 18 len 20480 data: 9 7 7 7 9
129802024-02-04T14:06:00.816Z16/30 Read block 390 len 12288
129812024-02-04T14:06:00.819Z17/30 Read block 18 len 4096
129822024-02-04T14:06:00.819Z18/30 Read block 250 len 40960
129832024-02-04T14:06:00.820Z19/30 Flush
129842024-02-04T14:06:00.823Z20/30 Write block 314 len 24576 data: 8 5 5 7 8 8
129852024-02-04T14:06:00.827Z21/30 Read block 473 len 36864
129862024-02-04T14:06:00.827Z22/30 Read block 202 len 32768
129872024-02-04T14:06:00.827Z23/30 Read block 384 len 16384
129882024-02-04T14:06:00.830Z24/30 Read block 569 len 40960
129892024-02-04T14:06:00.830Z25/30 Flush
129902024-02-04T14:06:00.833Z26/30 Write block 152 len 36864 data: 10 9 8 7 6 6 6 7 7
129912024-02-04T14:06:00.837Z27/30 Write block 122 len 4096 data: 9
129922024-02-04T14:06:00.838Z28/30 Read block 386 len 8192
129932024-02-04T14:06:00.838Z29/30 Read block 15 len 28672
129942024-02-04T14:06:00.842Z30/30 Write block 17 len 40960 data: 7 10 8 8 8 10 11 10 9 9
129952024-02-04T14:06:00.842Z----------------------------------------------------------------
129962024-02-04T14:06:00.842Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
129972024-02-04T14:06:00.842ZStates: Active Active Active
129982024-02-04T14:06:00.842ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129992024-02-04T14:06:00.842Z 1 Acked 1000 Write 4 Done Done Done false
130002024-02-04T14:06:00.842Z 2 Acked 1001 Write 6 Done Sent Done false
130012024-02-04T14:06:00.842Z 3 Acked 1002 Write 8 Done Sent Done false
130022024-02-04T14:06:00.842Z 4 Acked 1003 Read 1 Done Sent Done false
130032024-02-04T14:06:00.843Z 5 Acked 1004 Read 5 Done Sent Done false
130042024-02-04T14:06:00.843Z 6 Acked 1005 Write 1 Done Sent Done false
130052024-02-04T14:06:00.843Z 7 Acked 1006 Write 6 Done Sent Done false
130062024-02-04T14:06:00.843Z 8 Acked 1007 Write 3 Done Sent Done false
130072024-02-04T14:06:00.843Z 9 Acked 1008 Read 9 Done Sent Done false
130082024-02-04T14:06:00.843Z 10 Acked 1009 Flush 0 Done Sent Done false
130092024-02-04T14:06:00.843Z 11 Acked 1010 Flush 0 Done Sent Done false
130102024-02-04T14:06:00.843Z 12 Acked 1011 Flush 0 Done Sent Done false
130112024-02-04T14:06:00.843Z 13 Acked 1012 Write 6 Done Sent Done false
130122024-02-04T14:06:00.843Z 14 Acked 1013 Read 7 Done Sent Done false
130132024-02-04T14:06:00.843Z 15 Acked 1014 Write 5 Done Sent Done false
130142024-02-04T14:06:00.843Z 16 Acked 1015 Read 3 Done Sent Done false
130152024-02-04T14:06:00.843Z 17 Acked 1016 Read 1 Done Sent Done false
130162024-02-04T14:06:00.844Z 18 Acked 1017 Read 10 Done Sent Done false
130172024-02-04T14:06:00.844Z 19 Acked 1018 Flush 0 Done Sent Done false
130182024-02-04T14:06:00.844Z 20 Acked 1019 Write 6 Done Sent Done false
130192024-02-04T14:06:00.844Z 21 Acked 1020 Read 9 Done Sent Done false
130202024-02-04T14:06:00.844Z 22 Acked 1021 Read 8 Done Sent Done false
130212024-02-04T14:06:00.844Z 23 Acked 1022 Read 4 Done Sent Done false
130222024-02-04T14:06:00.844Z 24 Acked 1023 Read 10 Done Sent Done false
130232024-02-04T14:06:00.844Z 25 Acked 1024 Flush 0 Done Sent Done false
130242024-02-04T14:06:00.844Z 26 Acked 1025 Write 9 Done Sent Done false
130252024-02-04T14:06:00.844Z 27 Acked 1026 Write 1 Done Sent Done false
130262024-02-04T14:06:00.844Z 28 Acked 1027 Read 2 Done Sent Done false
130272024-02-04T14:06:00.844Z 29 Acked 1028 Read 7 Done Sent Done false
130282024-02-04T14:06:00.845Z 30 Acked 1029 Write 10 Sent Sent Sent false
130292024-02-04T14:06:00.845Z STATES DS:0 DS:1 DS:2 TOTAL
130302024-02-04T14:06:00.845Z New 0 0 0 0
130312024-02-04T14:06:00.845Z Sent 1 29 1 31
130322024-02-04T14:06:00.845Z Done 29 1 29 59
130332024-02-04T14:06:00.845Z Skipped 0 0 0 0
130342024-02-04T14:06:00.845Z Error 0 0 0 0
130352024-02-04T14:06:00.845ZLast Flush: 1024 0 1024
130362024-02-04T14:06:00.845ZDownstairs last five completed:
130372024-02-04T14:06:00.845ZUpstairs last five completed: 30 29 28 27 26
130382024-02-04T14:06:00.845Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:05:59.873615703Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1609}
130392024-02-04T14:06:00.845ZWrote out file "/tmp/test_repair_verify.data"
130402024-02-04T14:06:00.850Z
130412024-02-04T14:06:00.853ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130422024-02-04T14:06:00.878ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130432024-02-04T14:06:00.878Z 0 000-019 120 118 120 199 194 199 T F T <---
130442024-02-04T14:06:00.878Z 1 020-039 120 119 120 199 195 199 T F T <---
130452024-02-04T14:06:00.878Z 2 040-059 113 113 113 185 185 185 F F F
130462024-02-04T14:06:00.878Z 3 060-079 120 119 120 199 195 199 F F F <---
130472024-02-04T14:06:00.881Z 4 080-099 112 112 112 182 182 182 F F F
130482024-02-04T14:06:00.881Z 5 100-119 120 112 120 196 182 196 F F F <---
130492024-02-04T14:06:00.882Z 6 120-139 113 113 113 185 185 185 T F T <---
130502024-02-04T14:06:00.882Z 7 140-159 116 116 116 189 189 189 T F T <---
130512024-02-04T14:06:00.882Z 8 160-179 115 115 115 187 187 187 T F T <---
130522024-02-04T14:06:00.882Z 9 180-199 119 119 119 195 195 195 F F F
130532024-02-04T14:06:00.882Z 10 200-219 109 109 109 178 178 178 F F F
130542024-02-04T14:06:00.882Z 11 220-239 120 118 120 196 194 196 F F F <---
130552024-02-04T14:06:00.882Z 12 240-259 120 115 120 196 187 196 F T F <---
130562024-02-04T14:06:00.882Z 13 260-279 116 116 116 189 189 189 F F F
130572024-02-04T14:06:00.882Z 14 280-299 120 119 120 196 195 196 F F F <---
130582024-02-04T14:06:00.882Z 15 300-319 120 115 120 200 187 200 F F F <---
130592024-02-04T14:06:00.882Z 16 320-339 116 116 116 189 189 189 F F F
130602024-02-04T14:06:00.882Z 17 340-359 118 118 118 194 194 194 F F F
130612024-02-04T14:06:00.883Z 18 360-379 116 116 116 188 188 188 F F F
130622024-02-04T14:06:00.883Z 19 380-399 115 115 115 187 187 187 F F F
130632024-02-04T14:06:00.883Z 20 400-419 117 117 117 192 192 192 F F F
130642024-02-04T14:06:00.883Z 21 420-439 117 117 117 192 192 192 F F F
130652024-02-04T14:06:00.883Z 22 440-459 111 111 111 181 181 181 F F F
130662024-02-04T14:06:00.883Z 23 460-479 116 116 116 191 191 191 F F F
130672024-02-04T14:06:00.883Z 24 480-499 120 116 120 196 191 196 F F F <---
130682024-02-04T14:06:00.883Z 25 500-519 113 113 113 185 185 185 F F F
130692024-02-04T14:06:00.883Z 26 520-539 116 116 116 188 188 188 F F F
130702024-02-04T14:06:00.883Z 27 540-559 118 118 118 194 194 194 F F F
130712024-02-04T14:06:00.883Z 28 560-579 118 118 118 194 194 194 F F F
130722024-02-04T14:06:00.883Z 29 580-599 119 119 119 195 195 195 F F F
130732024-02-04T14:06:00.883ZMax gen: 120, Max flush: 200
130742024-02-04T14:06:00.884ZError: Difference in extent metadata found!
130752024-02-04T14:06:00.888ZA Difference in extent metadata is expected here
130762024-02-04T14:06:00.888ZOn loop 60
130772024-02-04T14:06:00.888Z
130782024-02-04T14:06:00.888ZVerifying data now
130792024-02-04T14:06:05.960ZNo change after verify
130802024-02-04T14:06:05.960ZLoop: 60 Downstairs dump after verify (and repair):
130812024-02-04T14:06:05.983ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130822024-02-04T14:06:05.983Z 0 000-019 121 121 121 201 201 201 F F F
130832024-02-04T14:06:05.983Z 1 020-039 121 121 121 201 201 201 F F F
130842024-02-04T14:06:05.983Z 2 040-059 113 113 113 185 185 185 F F F
130852024-02-04T14:06:05.983Z 3 060-079 120 120 120 199 199 199 F F F
130862024-02-04T14:06:05.986Z 4 080-099 112 112 112 182 182 182 F F F
130872024-02-04T14:06:05.987Z 5 100-119 120 120 120 196 196 196 F F F
130882024-02-04T14:06:05.987Z 6 120-139 121 121 121 201 201 201 F F F
130892024-02-04T14:06:05.987Z 7 140-159 121 121 121 201 201 201 F F F
130902024-02-04T14:06:05.987Z 8 160-179 121 121 121 201 201 201 F F F
130912024-02-04T14:06:05.987Z 9 180-199 119 119 119 195 195 195 F F F
130922024-02-04T14:06:05.987Z 10 200-219 109 109 109 178 178 178 F F F
130932024-02-04T14:06:05.987Z 11 220-239 120 120 120 196 196 196 F F F
130942024-02-04T14:06:05.987Z 12 240-259 120 120 120 196 196 196 F F F
130952024-02-04T14:06:05.987Z 13 260-279 116 116 116 189 189 189 F F F
130962024-02-04T14:06:05.987Z 14 280-299 120 120 120 196 196 196 F F F
130972024-02-04T14:06:05.987Z 15 300-319 120 120 120 200 200 200 F F F
130982024-02-04T14:06:05.987Z 16 320-339 116 116 116 189 189 189 F F F
130992024-02-04T14:06:05.987Z 17 340-359 118 118 118 194 194 194 F F F
131002024-02-04T14:06:05.987Z 18 360-379 116 116 116 188 188 188 F F F
131012024-02-04T14:06:05.987Z 19 380-399 115 115 115 187 187 187 F F F
131022024-02-04T14:06:05.987Z 20 400-419 117 117 117 192 192 192 F F F
131032024-02-04T14:06:05.987Z 21 420-439 117 117 117 192 192 192 F F F
131042024-02-04T14:06:05.987Z 22 440-459 111 111 111 181 181 181 F F F
131052024-02-04T14:06:05.987Z 23 460-479 116 116 116 191 191 191 F F F
131062024-02-04T14:06:05.987Z 24 480-499 120 120 120 196 196 196 F F F
131072024-02-04T14:06:05.987Z 25 500-519 113 113 113 185 185 185 F F F
131082024-02-04T14:06:05.987Z 26 520-539 116 116 116 188 188 188 F F F
131092024-02-04T14:06:05.987Z 27 540-559 118 118 118 194 194 194 F F F
131102024-02-04T14:06:05.988Z 28 560-579 118 118 118 194 194 194 F F F
131112024-02-04T14:06:05.988Z 29 580-599 119 119 119 195 195 195 F F F
131122024-02-04T14:06:05.988ZMax gen: 121, Max flush: 201
131132024-02-04T14:06:05.991Z
131142024-02-04T14:06:05.991ZBegin loop 61
131152024-02-04T14:06:05.991ZDownstairs to restart: 0
131162024-02-04T14:06:06.018Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.050605691Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617}
131172024-02-04T14:06:06.022Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051269303Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617}
131182024-02-04T14:06:06.022Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
131192024-02-04T14:06:06.022Z,"time":"2024-02-04T14:06:05.051342913Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617}
131202024-02-04T14:06:06.022Z{"msg":"Crucible 789e01a3-43a4-435e-a14d-2fc2bb421634 has session id: 23acf324-e62d-4f1d-b36e-b3e2425a5745","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051394751Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131212024-02-04T14:06:06.022Z{"msg":"Upstairs opts: Upstairs UUID: 789e01a3-43a4-435e-a14d-2fc2bb421634, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051447586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131222024-02-04T14:06:06.022Z{"msg":"Crucible stats registered with UUID: 789e01a3-43a4-435e-a14d-2fc2bb421634","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051518135Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131232024-02-04T14:06:06.022Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051585421Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617}
131242024-02-04T14:06:06.022Z{"msg":"789e01a3-43a4-435e-a14d-2fc2bb421634 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051778667Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131252024-02-04T14:06:06.022Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051855351Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131262024-02-04T14:06:06.022Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051927341Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131272024-02-04T14:06:06.022Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.051974152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131282024-02-04T14:06:06.022Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.05201973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131292024-02-04T14:06:06.022Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.052098452Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131302024-02-04T14:06:06.022Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.052146259Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131312024-02-04T14:06:06.022Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.052218592Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"io task","client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131322024-02-04T14:06:06.022Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.052360371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"io task","client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131332024-02-04T14:06:06.023Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.052437198Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"io task","client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131342024-02-04T14:06:06.023Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.05248718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"io task","client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131352024-02-04T14:06:06.023Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.052548137Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"io task","client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131362024-02-04T14:06:06.023Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.052575866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"io task","client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131372024-02-04T14:06:06.023Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.05342238Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131382024-02-04T14:06:06.023Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.053481353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131392024-02-04T14:06:06.023Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.053528325Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131402024-02-04T14:06:06.023Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.053695324Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131412024-02-04T14:06:06.023Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.054376695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131422024-02-04T14:06:06.023Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.054463526Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131432024-02-04T14:06:06.027Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.05451364Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131442024-02-04T14:06:06.027Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.054720524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131452024-02-04T14:06:06.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.055417243Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131462024-02-04T14:06:06.027Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.055479921Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131472024-02-04T14:06:06.027Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.055505757Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131482024-02-04T14:06:06.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.05553141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131492024-02-04T14:06:06.027Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.055595581Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131502024-02-04T14:06:06.028Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.055619799Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131512024-02-04T14:06:06.028Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.055642279Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131522024-02-04T14:06:06.028Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056364883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131532024-02-04T14:06:06.028Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056438218Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131542024-02-04T14:06:06.028Z{"msg":"[0]R flush_numbers[0..12]: [201, 201, 185, 199, 182, 196, 201, 201, 201, 195, 178, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056466985Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131552024-02-04T14:06:06.028Z{"msg":"[0]R generation[0..12]: [121, 121, 113, 120, 112, 120, 121, 121, 121, 119, 109, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056512124Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131562024-02-04T14:06:06.028Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056537928Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131572024-02-04T14:06:06.028Z{"msg":"[1]R flush_numbers[0..12]: [201, 201, 185, 199, 182, 196, 201, 201, 201, 195, 178, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056580066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131582024-02-04T14:06:06.028Z{"msg":"[1]R generation[0..12]: [121, 121, 113, 120, 112, 120, 121, 121, 121, 119, 109, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056604954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131592024-02-04T14:06:06.028Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056647915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131602024-02-04T14:06:06.028ZWait for a query_work_queue command to finish before sending IO
131612024-02-04T14:06:06.028Z{"msg":"[2]R flush_numbers[0..12]: [201, 201, 185, 199, 182, 196, 201, 201, 201, 195, 178, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056678097Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131622024-02-04T14:06:06.028Z{"msg":"[2]R generation[0..12]: [121, 121, 113, 120, 112, 120, 121, 121, 121, 119, 109, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056724788Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131632024-02-04T14:06:06.028Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056782305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131642024-02-04T14:06:06.028Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056847841Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131652024-02-04T14:06:06.028Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056872122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131662024-02-04T14:06:06.028Z{"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056894216Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131672024-02-04T14:06:06.028Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056955104Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131682024-02-04T14:06:06.028Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.056986176Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131692024-02-04T14:06:06.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.057059073Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131702024-02-04T14:06:06.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.057099711Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131712024-02-04T14:06:06.028Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.057159077Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131722024-02-04T14:06:06.028Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.057224537Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131732024-02-04T14:06:06.029Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.057251316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131742024-02-04T14:06:06.029Z{"msg":"789e01a3-43a4-435e-a14d-2fc2bb421634 is now active with session: 23acf324-e62d-4f1d-b36e-b3e2425a5745","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.057296342Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131752024-02-04T14:06:06.029Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.057317902Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617,"session_id":"23acf324-e62d-4f1d-b36e-b3e2425a5745"}
131762024-02-04T14:06:06.029Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:05.057370195Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617}
131772024-02-04T14:06:06.029ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131782024-02-04T14:06:06.029ZLoading write count information from file "/tmp/test_repair_verify.data"
131792024-02-04T14:06:06.029ZRun Repair workload
131802024-02-04T14:06:06.029Z01/30 Write block 209 len 16384 data: 5 5 7 8
131812024-02-04T14:06:06.032Z02/30 Read block 389 len 28672
131822024-02-04T14:06:06.032Z03/30 Flush
131832024-02-04T14:06:06.032Z04/30 Write block 176 len 20480 data: 10 13 12 14 13
131842024-02-04T14:06:06.035Z05/30 Write block 86 len 12288 data: 9 7 6
131852024-02-04T14:06:06.035Z06/30 Write block 228 len 4096 data: 11
131862024-02-04T14:06:06.035Z07/30 Read block 367 len 20480
131872024-02-04T14:06:06.038Z08/30 Read block 11 len 32768
131882024-02-04T14:06:06.038Z09/30 Flush
131892024-02-04T14:06:06.041Z10/30 Flush
131902024-02-04T14:06:06.041Z11/30 Flush
131912024-02-04T14:06:06.044Z12/30 Write block 6 len 12288 data: 8 8 8
131922024-02-04T14:06:06.044Z13/30 Write block 283 len 32768 data: 12 17 15 12 13 13 12 12
131932024-02-04T14:06:06.044Z14/30 Read block 235 len 24576
131942024-02-04T14:06:06.047Z15/30 Read block 300 len 36864
131952024-02-04T14:06:06.050Z16/30 Read block 10 len 36864
131962024-02-04T14:06:06.050Z17/30 Read block 407 len 24576
131972024-02-04T14:06:06.053Z18/30 Write block 498 len 16384 data: 13 13 14 14
131982024-02-04T14:06:06.053Z19/30 Write block 91 len 32768 data: 5 5 4 4 7 6 8 7
131992024-02-04T14:06:06.053Z20/30 Read block 593 len 12288
132002024-02-04T14:06:06.056Z21/30 Write block 209 len 24576 data: 6 6 8 9 8 10
132012024-02-04T14:06:06.056Z22/30 Read block 144 len 8192
132022024-02-04T14:06:06.059Z23/30 Read block 320 len 8192
132032024-02-04T14:06:06.059Z24/30 Read block 560 len 36864
132042024-02-04T14:06:06.062Z25/30 Read block 590 len 40960
132052024-02-04T14:06:06.062Z26/30 Read block 543 len 24576
132062024-02-04T14:06:06.065Z27/30 Read block 114 len 40960
132072024-02-04T14:06:06.068Z28/30 Read block 167 len 16384
132082024-02-04T14:06:06.068Z29/30 Read block 574 len 40960
132092024-02-04T14:06:06.084Z30/30 Read block 255 len 28672
132102024-02-04T14:06:06.084Z----------------------------------------------------------------
132112024-02-04T14:06:06.084Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
132122024-02-04T14:06:06.084ZStates: Active Active Active
132132024-02-04T14:06:06.085ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132142024-02-04T14:06:06.085Z 1 Acked 1000 Write 4 Sent Done Done false
132152024-02-04T14:06:06.085Z 2 Acked 1001 Read 7 Sent Done Done false
132162024-02-04T14:06:06.085Z 3 Acked 1002 Flush 0 Sent Done Done false
132172024-02-04T14:06:06.085Z 4 Acked 1003 Write 5 Sent Done Done false
132182024-02-04T14:06:06.085Z 5 Acked 1004 Write 3 Sent Done Done false
132192024-02-04T14:06:06.085Z 6 Acked 1005 Write 1 Sent Done Done false
132202024-02-04T14:06:06.085Z 7 Acked 1006 Read 5 Sent Done Done false
132212024-02-04T14:06:06.085Z 8 Acked 1007 Read 8 Sent Done Done false
132222024-02-04T14:06:06.085Z 9 Acked 1008 Flush 0 Sent Done Done false
132232024-02-04T14:06:06.085Z 10 Acked 1009 Flush 0 Sent Done Done false
132242024-02-04T14:06:06.085Z 11 Acked 1010 Flush 0 Sent Done Done false
132252024-02-04T14:06:06.085Z 12 Acked 1011 Write 3 Sent Done Done false
132262024-02-04T14:06:06.085Z 13 Acked 1012 Write 8 Sent Done Done false
132272024-02-04T14:06:06.085Z 14 Acked 1013 Read 6 Sent Done Done false
132282024-02-04T14:06:06.085Z 15 Acked 1014 Read 9 Sent Done Done false
132292024-02-04T14:06:06.085Z 16 Acked 1015 Read 9 Sent Done Done false
132302024-02-04T14:06:06.085Z 17 Acked 1016 Read 6 Sent Done Done false
132312024-02-04T14:06:06.085Z 18 Acked 1017 Write 4 Sent Done Done false
132322024-02-04T14:06:06.085Z 19 Acked 1018 Write 8 Sent Done Done false
132332024-02-04T14:06:06.085Z 20 Acked 1019 Read 3 Sent Done Done false
132342024-02-04T14:06:06.085Z 21 Acked 1020 Write 6 Sent Done Done false
132352024-02-04T14:06:06.085Z 22 Acked 1021 Read 2 Sent Done Done false
132362024-02-04T14:06:06.085Z 23 Acked 1022 Read 2 Sent Done Done false
132372024-02-04T14:06:06.086Z 24 Acked 1023 Read 9 Sent Done Done false
132382024-02-04T14:06:06.086Z 25 Acked 1024 Read 10 Sent Done Done false
132392024-02-04T14:06:06.086Z 26 Acked 1025 Read 6 Sent Done Done false
132402024-02-04T14:06:06.086Z 27 Acked 1026 Read 10 Sent Done Done false
132412024-02-04T14:06:06.086Z 28 Acked 1027 Read 4 Sent Done Done false
132422024-02-04T14:06:06.086Z 29 Acked 1028 Read 10 Sent Done Done false
132432024-02-04T14:06:06.086Z 30 Acked 1029 Read 7 Sent Done Done false
132442024-02-04T14:06:06.086Z STATES DS:0 DS:1 DS:2 TOTAL
132452024-02-04T14:06:06.086Z New 0 0 0 0
132462024-02-04T14:06:06.086Z Sent 30 0 0 30
132472024-02-04T14:06:06.086Z Done 0 30 30 60
132482024-02-04T14:06:06.086Z Skipped 0 0 0 0
132492024-02-04T14:06:06.086Z Error 0 0 0 0
132502024-02-04T14:06:06.086ZLast Flush: 0 1010 1010
132512024-02-04T14:06:06.086ZDownstairs last five completed:
132522024-02-04T14:06:06.086ZUpstairs last five completed: 30 29 28 27 26
132532024-02-04T14:06:06.086Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:05.103611767Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1617}
132542024-02-04T14:06:06.086ZWrote out file "/tmp/test_repair_verify.data"
132552024-02-04T14:06:06.090Z
132562024-02-04T14:06:06.090ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132572024-02-04T14:06:06.105ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132582024-02-04T14:06:06.105Z 0 000-019 121 121 121 201 201 201 F T T <---
132592024-02-04T14:06:06.106Z 1 020-039 121 121 121 201 201 201 F F F
132602024-02-04T14:06:06.106Z 2 040-059 113 113 113 185 185 185 F F F
132612024-02-04T14:06:06.106Z 3 060-079 120 120 120 199 199 199 F F F
132622024-02-04T14:06:06.106Z 4 080-099 112 122 122 182 203 203 F T T <---
132632024-02-04T14:06:06.106Z 5 100-119 120 120 120 196 196 196 F F F
132642024-02-04T14:06:06.106Z 6 120-139 121 121 121 201 201 201 F F F
132652024-02-04T14:06:06.109Z 7 140-159 121 121 121 201 201 201 F F F
132662024-02-04T14:06:06.109Z 8 160-179 121 122 122 201 203 203 F F F <---
132672024-02-04T14:06:06.109Z 9 180-199 119 122 122 195 203 203 F F F <---
132682024-02-04T14:06:06.109Z 10 200-219 109 122 122 178 202 202 F T T <---
132692024-02-04T14:06:06.109Z 11 220-239 120 122 122 196 203 203 F F F <---
132702024-02-04T14:06:06.109Z 12 240-259 120 120 120 196 196 196 F F F
132712024-02-04T14:06:06.109Z 13 260-279 116 116 116 189 189 189 F F F
132722024-02-04T14:06:06.109Z 14 280-299 120 120 120 196 196 196 F T T <---
132732024-02-04T14:06:06.109Z 15 300-319 120 120 120 200 200 200 F F F
132742024-02-04T14:06:06.109Z 16 320-339 116 116 116 189 189 189 F F F
132752024-02-04T14:06:06.109Z 17 340-359 118 118 118 194 194 194 F F F
132762024-02-04T14:06:06.109Z 18 360-379 116 116 116 188 188 188 F F F
132772024-02-04T14:06:06.109Z 19 380-399 115 115 115 187 187 187 F F F
132782024-02-04T14:06:06.109Z 20 400-419 117 117 117 192 192 192 F F F
132792024-02-04T14:06:06.109Z 21 420-439 117 117 117 192 192 192 F F F
132802024-02-04T14:06:06.109Z 22 440-459 111 111 111 181 181 181 F F F
132812024-02-04T14:06:06.109Z 23 460-479 116 116 116 191 191 191 F F F
132822024-02-04T14:06:06.110Z 24 480-499 120 120 120 196 196 196 F T T <---
132832024-02-04T14:06:06.110Z 25 500-519 113 113 113 185 185 185 F T T <---
132842024-02-04T14:06:06.110Z 26 520-539 116 116 116 188 188 188 F F F
132852024-02-04T14:06:06.110Z 27 540-559 118 118 118 194 194 194 F F F
132862024-02-04T14:06:06.110Z 28 560-579 118 118 118 194 194 194 F F F
132872024-02-04T14:06:06.110Z 29 580-599 119 119 119 195 195 195 F F F
132882024-02-04T14:06:06.110ZMax gen: 122, Max flush: 203
132892024-02-04T14:06:06.110ZError: Difference in extent metadata found!
132902024-02-04T14:06:06.113ZA Difference in extent metadata is expected here
132912024-02-04T14:06:06.113ZOn loop 61
132922024-02-04T14:06:06.113Z
132932024-02-04T14:06:06.116ZVerifying data now
132942024-02-04T14:06:10.950ZNo change after verify
132952024-02-04T14:06:10.950ZLoop: 61 Downstairs dump after verify (and repair):
132962024-02-04T14:06:10.972ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132972024-02-04T14:06:10.972Z 0 000-019 123 123 123 204 204 204 F F F
132982024-02-04T14:06:10.972Z 1 020-039 121 121 121 201 201 201 F F F
132992024-02-04T14:06:10.972Z 2 040-059 113 113 113 185 185 185 F F F
133002024-02-04T14:06:10.972Z 3 060-079 120 120 120 199 199 199 F F F
133012024-02-04T14:06:10.972Z 4 080-099 123 123 123 204 204 204 F F F
133022024-02-04T14:06:10.972Z 5 100-119 120 120 120 196 196 196 F F F
133032024-02-04T14:06:10.995Z 6 120-139 121 121 121 201 201 201 F F F
133042024-02-04T14:06:10.995Z 7 140-159 121 121 121 201 201 201 F F F
133052024-02-04T14:06:10.995Z 8 160-179 122 122 122 203 203 203 F F F
133062024-02-04T14:06:10.996Z 9 180-199 122 122 122 203 203 203 F F F
133072024-02-04T14:06:10.996Z 10 200-219 123 123 123 204 204 204 F F F
133082024-02-04T14:06:10.996Z 11 220-239 122 122 122 203 203 203 F F F
133092024-02-04T14:06:10.996Z 12 240-259 120 120 120 196 196 196 F F F
133102024-02-04T14:06:10.996Z 13 260-279 116 116 116 189 189 189 F F F
133112024-02-04T14:06:10.996Z 14 280-299 123 123 123 204 204 204 F F F
133122024-02-04T14:06:10.996Z 15 300-319 120 120 120 200 200 200 F F F
133132024-02-04T14:06:10.996Z 16 320-339 116 116 116 189 189 189 F F F
133142024-02-04T14:06:10.996Z 17 340-359 118 118 118 194 194 194 F F F
133152024-02-04T14:06:10.996Z 18 360-379 116 116 116 188 188 188 F F F
133162024-02-04T14:06:10.996Z 19 380-399 115 115 115 187 187 187 F F F
133172024-02-04T14:06:10.996Z 20 400-419 117 117 117 192 192 192 F F F
133182024-02-04T14:06:10.996Z 21 420-439 117 117 117 192 192 192 F F F
133192024-02-04T14:06:10.996Z 22 440-459 111 111 111 181 181 181 F F F
133202024-02-04T14:06:10.996Z 23 460-479 116 116 116 191 191 191 F F F
133212024-02-04T14:06:10.996Z 24 480-499 123 123 123 204 204 204 F F F
133222024-02-04T14:06:10.996Z 25 500-519 123 123 123 204 204 204 F F F
133232024-02-04T14:06:10.996Z 26 520-539 116 116 116 188 188 188 F F F
133242024-02-04T14:06:10.996Z 27 540-559 118 118 118 194 194 194 F F F
133252024-02-04T14:06:10.996Z 28 560-579 118 118 118 194 194 194 F F F
133262024-02-04T14:06:10.996Z 29 580-599 119 119 119 195 195 195 F F F
133272024-02-04T14:06:10.996ZMax gen: 123, Max flush: 204
133282024-02-04T14:06:10.996Z
133292024-02-04T14:06:10.996ZBegin loop 62
133302024-02-04T14:06:10.997ZDownstairs to restart: 1
133312024-02-04T14:06:11.007Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.039645029Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625}
133322024-02-04T14:06:11.011Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040153785Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625}
133332024-02-04T14:06:11.011Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040242045Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625}
133342024-02-04T14:06:11.011Z{"msg":"Crucible 1b462983-1ccb-4181-a80f-9a477cd11fac has session id: 735d5ace-efe6-446b-92d1-8fcd7192e102","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
133352024-02-04T14:06:11.011Z2024-02-04T14:06:10.04028314Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133362024-02-04T14:06:11.011Z{"msg":"Upstairs opts: Upstairs UUID: 1b462983-1ccb-4181-a80f-9a477cd11fac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040521481Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133372024-02-04T14:06:11.011Z{"msg":"Crucible stats registered with UUID: 1b462983-1ccb-4181-a80f-9a477cd11fac","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040586929Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133382024-02-04T14:06:11.011Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040629181Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625}
133392024-02-04T14:06:11.011Z{"msg":"1b462983-1ccb-4181-a80f-9a477cd11fac active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040680274Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133402024-02-04T14:06:11.011Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040719896Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133412024-02-04T14:06:11.011Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040774736Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133422024-02-04T14:06:11.011Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040816955Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133432024-02-04T14:06:11.011Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040856711Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133442024-02-04T14:06:11.011Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.040942233Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133452024-02-04T14:06:11.011Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.041042493Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133462024-02-04T14:06:11.011Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.041092899Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133472024-02-04T14:06:11.011Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.041215924Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133482024-02-04T14:06:11.011Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.041310022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133492024-02-04T14:06:11.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.041397574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133502024-02-04T14:06:11.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.041443845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133512024-02-04T14:06:11.011Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.041516711Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133522024-02-04T14:06:11.012Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.042542333Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133532024-02-04T14:06:11.012Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.042628966Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133542024-02-04T14:06:11.012Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.042861122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133552024-02-04T14:06:11.012Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.042946755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133562024-02-04T14:06:11.012Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.043078962Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133572024-02-04T14:06:11.016Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.043238068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133582024-02-04T14:06:11.016Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.043622528Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133592024-02-04T14:06:11.016Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.043981821Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133602024-02-04T14:06:11.016Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.04421392Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133612024-02-04T14:06:11.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.044441664Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133622024-02-04T14:06:11.016Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.044523534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133632024-02-04T14:06:11.016Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.04456503Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133642024-02-04T14:06:11.016Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.044801967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133652024-02-04T14:06:11.016Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.044881139Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133662024-02-04T14:06:11.017Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.044911189Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133672024-02-04T14:06:11.017Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.045152305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133682024-02-04T14:06:11.017Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.045235915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133692024-02-04T14:06:11.017Z{"msg":"[0]R flush_numbers[0..12]: [204, 201, 185, 199, 204, 196, 201, 201, 203, 203, 204, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.045313567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133702024-02-04T14:06:11.017Z{"msg":"[0]R generation[0..12]: [123, 121, 113, 120, 123, 120, 121, 121, 122, 122, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.04538749Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133712024-02-04T14:06:11.017Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.045466557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133722024-02-04T14:06:11.017Z{"msg":"[1]R flush_numbers[0..12]: [204, 201, 185, 199, 204, 196, 201, 201, 203, 203, 204, 203]","Wait for a query_work_queue command to finish before sending IO
133732024-02-04T14:06:11.017Zv":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.045636863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133742024-02-04T14:06:11.017Z{"msg":"[1]R generation[0..12]: [123, 121, 113, 120, 123, 120, 121, 121, 122, 122, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.04571626Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133752024-02-04T14:06:11.017Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.04579514Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133762024-02-04T14:06:11.017Z{"msg":"[2]R flush_numbers[0..12]: [204, 201, 185, 199, 204, 196, 201, 201, 203, 203, 204, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.045870855Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133772024-02-04T14:06:11.017Z{"msg":"[2]R generation[0..12]: [123, 121, 113, 120, 123, 120, 121, 121, 122, 122, 123, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.045943887Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133782024-02-04T14:06:11.017Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.04602323Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133792024-02-04T14:06:11.017Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046105471Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133802024-02-04T14:06:11.017Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046176039Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133812024-02-04T14:06:11.017Z{"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046222482Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133822024-02-04T14:06:11.017Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046276623Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133832024-02-04T14:06:11.017Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046301903Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133842024-02-04T14:06:11.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046322648Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133852024-02-04T14:06:11.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046364889Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133862024-02-04T14:06:11.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046388535Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133872024-02-04T14:06:11.017Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046411068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133882024-02-04T14:06:11.017Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046458229Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133892024-02-04T14:06:11.017Z{"msg":"1b462983-1ccb-4181-a80f-9a477cd11fac is now active with session: 735d5ace-efe6-446b-92d1-8fcd7192e102","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046481688Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133902024-02-04T14:06:11.018Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046502081Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625,"session_id":"735d5ace-efe6-446b-92d1-8fcd7192e102"}
133912024-02-04T14:06:11.018Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:10.046541837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625}
133922024-02-04T14:06:11.018ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133932024-02-04T14:06:11.018ZLoading write count information from file "/tmp/test_repair_verify.data"
133942024-02-04T14:06:11.018ZRun Repair workload
133952024-02-04T14:06:11.018Z01/30 Write block 582 len 32768 data: 12 13 14 13 13 11 12 11
133962024-02-04T14:06:11.021Z02/30 Read block 536 len 24576
133972024-02-04T14:06:11.021Z03/30 Write block 550 len 28672 data: 7 5 5 9 9 9 11
133982024-02-04T14:06:11.021Z04/30 Read block 566 len 4096
133992024-02-04T14:06:11.024Z05/30 Write block 349 len 24576 data: 13 10 12 12 13 11
134002024-02-04T14:06:11.024Z06/30 Write block 268 len 12288 data: 11 10 9
134012024-02-04T14:06:11.027Z07/30 Write block 234 len 16384 data: 10 8 9 11
134022024-02-04T14:06:11.027Z08/30 Read block 430 len 4096
134032024-02-04T14:06:11.027Z09/30 Read block 138 len 32768
134042024-02-04T14:06:11.031Z10/30 Write block 488 len 8192 data: 10 13
134052024-02-04T14:06:11.031Z11/30 Write block 80 len 28672 data: 11 12 13 11 11 11 10
134062024-02-04T14:06:11.031Z12/30 Write block 233 len 20480 data: 9 11 9 10 12
134072024-02-04T14:06:11.034Z13/30 Read block 80 len 4096
134082024-02-04T14:06:11.034Z14/30 Read block 225 len 16384
134092024-02-04T14:06:11.037Z15/30 Write block 132 len 4096 data: 7
134102024-02-04T14:06:11.037Z16/30 Flush
134112024-02-04T14:06:11.040Z17/30 Write block 225 len 40960 data: 9 9 9 12 10 9 8 8 10 12
134122024-02-04T14:06:11.040Z18/30 Read block 56 len 28672
134132024-02-04T14:06:11.042Z19/30 Read block 117 len 24576
134142024-02-04T14:06:11.045Z20/30 Read block 175 len 20480
134152024-02-04T14:06:11.048Z21/30 Write block 558 len 32768 data: 11 13 14 11 10 11 13 14
134162024-02-04T14:06:11.048Z22/30 Write block 444 len 28672 data: 11 11 12 12 11 10 10
134172024-02-04T14:06:11.048Z23/30 Read block 373 len 16384
134182024-02-04T14:06:11.052Z24/30 Write block 526 len 12288 data: 11 10 11
134192024-02-04T14:06:11.052Z25/30 Read block 546 len 24576
134202024-02-04T14:06:11.054Z26/30 Write block 383 len 20480 data: 7 8 8 7 9
134212024-02-04T14:06:11.055Z27/30 Read block 249 len 36864
134222024-02-04T14:06:11.058Z28/30 Write block 507 len 16384 data: 15 14 14 12
134232024-02-04T14:06:11.058Z29/30 Write block 61 len 8192 data: 10 11
134242024-02-04T14:06:11.058Z30/30 Read block 61 len 8192
134252024-02-04T14:06:11.061Z----------------------------------------------------------------
134262024-02-04T14:06:11.061Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
134272024-02-04T14:06:11.061ZStates: Active Active Active
134282024-02-04T14:06:11.061ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134292024-02-04T14:06:11.061Z 1 Acked 1000 Write 8 Done Done Done false
134302024-02-04T14:06:11.061Z 2 Acked 1001 Read 6 Done Done Done false
134312024-02-04T14:06:11.061Z 3 Acked 1002 Write 7 Done Done Done false
134322024-02-04T14:06:11.061Z 4 Acked 1003 Read 1 Done Done Done false
134332024-02-04T14:06:11.061Z 5 Acked 1004 Write 6 Done Done Done false
134342024-02-04T14:06:11.061Z 6 Acked 1005 Write 3 Done Done Done false
134352024-02-04T14:06:11.061Z 7 Acked 1006 Write 4 Done Sent Done false
134362024-02-04T14:06:11.062Z 8 Acked 1007 Read 1 Done Sent Done false
134372024-02-04T14:06:11.062Z 9 Acked 1008 Read 8 Done Sent Done false
134382024-02-04T14:06:11.062Z 10 Acked 1009 Write 2 Done Sent Done false
134392024-02-04T14:06:11.062Z 11 Acked 1010 Write 7 Done Sent Done false
134402024-02-04T14:06:11.062Z 12 Acked 1011 Write 5 Done Sent Done false
134412024-02-04T14:06:11.062Z 13 Acked 1012 Read 1 Done Sent Done false
134422024-02-04T14:06:11.062Z 14 Acked 1013 Read 4 Done Sent Done false
134432024-02-04T14:06:11.062Z 15 Acked 1014 Write 1 Done Sent Done false
134442024-02-04T14:06:11.062Z 16 Acked 1015 Flush 0 Done Sent Done false
134452024-02-04T14:06:11.062Z 17 Acked 1016 Write 10 Done Sent Done false
134462024-02-04T14:06:11.062Z 18 Acked 1017 Read 7 Done Sent Done false
134472024-02-04T14:06:11.062Z 19 Acked 1018 Read 6 Done Sent Done false
134482024-02-04T14:06:11.062Z 20 Acked 1019 Read 5 Done Sent Done false
134492024-02-04T14:06:11.062Z 21 Acked 1020 Write 8 Done Sent Done false
134502024-02-04T14:06:11.062Z 22 Acked 1021 Write 7 Done Sent Done false
134512024-02-04T14:06:11.062Z 23 Acked 1022 Read 4 Done Sent Done false
134522024-02-04T14:06:11.062Z 24 Acked 1023 Write 3 Done Sent Done false
134532024-02-04T14:06:11.062Z 25 Acked 1024 Read 6 Done Sent Done false
134542024-02-04T14:06:11.062Z 26 Acked 1025 Write 5 Done Sent Done false
134552024-02-04T14:06:11.062Z 27 Acked 1026 Read 9 Done Sent Done false
134562024-02-04T14:06:11.062Z 28 Acked 1027 Write 4 Done Sent Done false
134572024-02-04T14:06:11.062Z 29 Acked 1028 Write 2 Done Sent Done false
134582024-02-04T14:06:11.062Z 30 Acked 1029 Read 2 Done Sent Done false
134592024-02-04T14:06:11.062Z STATES DS:0 DS:1 DS:2 TOTAL
134602024-02-04T14:06:11.063Z New 0 0 0 0
134612024-02-04T14:06:11.063Z Sent 0 24 0 24
134622024-02-04T14:06:11.063Z Done 30 6 30 66
134632024-02-04T14:06:11.063Z Skipped 0 0 0 0
134642024-02-04T14:06:11.063Z Error 0 0 0 0
134652024-02-04T14:06:11.063ZLast Flush: 1015 0 1015
134662024-02-04T14:06:11.063ZDownstairs last five completed:
134672024-02-04T14:06:11.063ZUpstairs last five completed: 30 29 28 27 26
134682024-02-04T14:06:11.063Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:10.09194072Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1625}
134692024-02-04T14:06:11.063ZWrote out file "/tmp/test_repair_verify.data"
134702024-02-04T14:06:11.066Z
134712024-02-04T14:06:11.071ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134722024-02-04T14:06:11.099ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134732024-02-04T14:06:11.099Z 0 000-019 123 123 123 204 204 204 F F F
134742024-02-04T14:06:11.099Z 1 020-039 121 121 121 201 201 201 F F F
134752024-02-04T14:06:11.099Z 2 040-059 113 113 113 185 185 185 F F F
134762024-02-04T14:06:11.099Z 3 060-079 120 120 120 199 199 199 T F T <---
134772024-02-04T14:06:11.099Z 4 080-099 124 123 124 205 204 205 F F F <---
134782024-02-04T14:06:11.099Z 5 100-119 120 120 120 196 196 196 F F F
134792024-02-04T14:06:11.099Z 6 120-139 124 121 124 205 201 205 F F F <---
134802024-02-04T14:06:11.102Z 7 140-159 121 121 121 201 201 201 F F F
134812024-02-04T14:06:11.102Z 8 160-179 122 122 122 203 203 203 F F F
134822024-02-04T14:06:11.103Z 9 180-199 122 122 122 203 203 203 F F F
134832024-02-04T14:06:11.103Z 10 200-219 123 123 123 204 204 204 F F F
134842024-02-04T14:06:11.103Z 11 220-239 124 122 124 205 203 205 T F T <---
134852024-02-04T14:06:11.103Z 12 240-259 120 120 120 196 196 196 F F F
134862024-02-04T14:06:11.103Z 13 260-279 124 116 124 205 189 205 F T F <---
134872024-02-04T14:06:11.103Z 14 280-299 123 123 123 204 204 204 F F F
134882024-02-04T14:06:11.103Z 15 300-319 120 120 120 200 200 200 F F F
134892024-02-04T14:06:11.103Z 16 320-339 116 116 116 189 189 189 F F F
134902024-02-04T14:06:11.103Z 17 340-359 124 118 124 205 194 205 F T F <---
134912024-02-04T14:06:11.103Z 18 360-379 116 116 116 188 188 188 F F F
134922024-02-04T14:06:11.103Z 19 380-399 115 115 115 187 187 187 T F T <---
134932024-02-04T14:06:11.103Z 20 400-419 117 117 117 192 192 192 F F F
134942024-02-04T14:06:11.103Z 21 420-439 117 117 117 192 192 192 F F F
134952024-02-04T14:06:11.103Z 22 440-459 111 111 111 181 181 181 T F T <---
134962024-02-04T14:06:11.103Z 23 460-479 116 116 116 191 191 191 F F F
134972024-02-04T14:06:11.103Z 24 480-499 124 123 124 205 204 205 F F F <---
134982024-02-04T14:06:11.103Z 25 500-519 123 123 123 204 204 204 T F T <---
134992024-02-04T14:06:11.103Z 26 520-539 116 116 116 188 188 188 T F T <---
135002024-02-04T14:06:11.103Z 27 540-559 124 118 124 205 194 205 T T T <---
135012024-02-04T14:06:11.103Z 28 560-579 118 118 118 194 194 194 T F T <---
135022024-02-04T14:06:11.103Z 29 580-599 124 119 124 205 195 205 F T F <---
135032024-02-04T14:06:11.103ZMax gen: 124, Max flush: 205
135042024-02-04T14:06:11.103ZError: Difference in extent metadata found!
135052024-02-04T14:06:11.107ZA Difference in extent metadata is expected here
135062024-02-04T14:06:11.107ZOn loop 62
135072024-02-04T14:06:11.107Z
135082024-02-04T14:06:11.107ZVerifying data now
135092024-02-04T14:06:17.316ZNo change after verify
135102024-02-04T14:06:17.316ZLoop: 62 Downstairs dump after verify (and repair):
135112024-02-04T14:06:17.356ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135122024-02-04T14:06:17.356Z 0 000-019 123 123 123 204 204 204 F F F
135132024-02-04T14:06:17.356Z 1 020-039 121 121 121 201 201 201 F F F
135142024-02-04T14:06:17.357Z 2 040-059 113 113 113 185 185 185 F F F
135152024-02-04T14:06:17.357Z 3 060-079 125 125 125 206 206 206 F F F
135162024-02-04T14:06:17.357Z 4 080-099 124 124 124 205 205 205 F F F
135172024-02-04T14:06:17.357Z 5 100-119 120 120 120 196 196 196 F F F
135182024-02-04T14:06:17.357Z 6 120-139 124 124 124 205 205 205 F F F
135192024-02-04T14:06:17.357Z 7 140-159 121 121 121 201 201 201 F F F
135202024-02-04T14:06:17.365Z 8 160-179 122 122 122 203 203 203 F F F
135212024-02-04T14:06:17.365Z 9 180-199 122 122 122 203 203 203 F F F
135222024-02-04T14:06:17.365Z 10 200-219 123 123 123 204 204 204 F F F
135232024-02-04T14:06:17.366Z 11 220-239 125 125 125 206 206 206 F F F
135242024-02-04T14:06:17.366Z 12 240-259 120 120 120 196 196 196 F F F
135252024-02-04T14:06:17.366Z 13 260-279 124 124 124 205 205 205 F F F
135262024-02-04T14:06:17.366Z 14 280-299 123 123 123 204 204 204 F F F
135272024-02-04T14:06:17.366Z 15 300-319 120 120 120 200 200 200 F F F
135282024-02-04T14:06:17.367Z 16 320-339 116 116 116 189 189 189 F F F
135292024-02-04T14:06:17.367Z 17 340-359 124 124 124 205 205 205 F F F
135302024-02-04T14:06:17.367Z 18 360-379 116 116 116 188 188 188 F F F
135312024-02-04T14:06:17.367Z 19 380-399 125 125 125 206 206 206 F F F
135322024-02-04T14:06:17.367Z 20 400-419 117 117 117 192 192 192 F F F
135332024-02-04T14:06:17.367Z 21 420-439 117 117 117 192 192 192 F F F
135342024-02-04T14:06:17.368Z 22 440-459 125 125 125 206 206 206 F F F
135352024-02-04T14:06:17.368Z 23 460-479 116 116 116 191 191 191 F F F
135362024-02-04T14:06:17.368Z 24 480-499 124 124 124 205 205 205 F F F
135372024-02-04T14:06:17.368Z 25 500-519 125 125 125 206 206 206 F F F
135382024-02-04T14:06:17.368Z 26 520-539 125 125 125 206 206 206 F F F
135392024-02-04T14:06:17.368Z 27 540-559 125 125 125 206 206 206 F F F
135402024-02-04T14:06:17.369Z 28 560-579 125 125 125 206 206 206 F F F
135412024-02-04T14:06:17.369Z 29 580-599 124 124 124 205 205 205 F F F
135422024-02-04T14:06:17.369ZMax gen: 125, Max flush: 206
135432024-02-04T14:06:17.369Z
135442024-02-04T14:06:17.369ZBegin loop 63
135452024-02-04T14:06:17.369ZDownstairs to restart: 2
135462024-02-04T14:06:17.382Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.408969697Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633}
135472024-02-04T14:06:17.413Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.409585882Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633}
135482024-02-04T14:06:17.413Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.409657947Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633}
135492024-02-04T14:06:17.413Z{Crucible runtime is spawned
135502024-02-04T14:06:17.413Z"msg":"Crucible 79fcdc01-79ac-4254-9be1-d07a55ce319f has session id: cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.409777846Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135512024-02-04T14:06:17.414Z{"msg":"Upstairs opts: Upstairs UUID: 79fcdc01-79ac-4254-9be1-d07a55ce319f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.409849725Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135522024-02-04T14:06:17.414Z{"msg":"Crucible stats registered with UUID: 79fcdc01-79ac-4254-9be1-d07a55ce319f","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.409913197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135532024-02-04T14:06:17.414Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.409954091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633}
135542024-02-04T14:06:17.414Z{"msg":"79fcdc01-79ac-4254-9be1-d07a55ce319f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410003667Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135552024-02-04T14:06:17.414Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410042015Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135562024-02-04T14:06:17.414Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410136013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135572024-02-04T14:06:17.415Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410178173Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135582024-02-04T14:06:17.415Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410215906Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135592024-02-04T14:06:17.415Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410361884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135602024-02-04T14:06:17.415Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410463867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135612024-02-04T14:06:17.415Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.41051051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"io task","client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135622024-02-04T14:06:17.415Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410596948Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"io task","client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135632024-02-04T14:06:17.415Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410633706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"io task","client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135642024-02-04T14:06:17.416Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.41065829Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"io task","client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135652024-02-04T14:06:17.416Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410713128Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"io task","client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135662024-02-04T14:06:17.416Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.410744555Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"io task","client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135672024-02-04T14:06:17.416Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.411904948Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135682024-02-04T14:06:17.416Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.411973504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135692024-02-04T14:06:17.416Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.411999303Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135702024-02-04T14:06:17.416Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.412021922Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135712024-02-04T14:06:17.417Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.412896696Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135722024-02-04T14:06:17.417Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.413111261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135732024-02-04T14:06:17.417Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.413302771Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135742024-02-04T14:06:17.417Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.413385055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135752024-02-04T14:06:17.417Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.413830217Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135762024-02-04T14:06:17.418Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.413890232Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135772024-02-04T14:06:17.418Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.413915526Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135782024-02-04T14:06:17.418Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.413937288Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135792024-02-04T14:06:17.419Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.413979457Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135802024-02-04T14:06:17.419Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.414009391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135812024-02-04T14:06:17.419Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.414182068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135822024-02-04T14:06:17.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.414846234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135832024-02-04T14:06:17.419Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.414931122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135842024-02-04T14:06:17.420Z{"msg":"[0]R flush_numbers[0..12]: [204, 201, 185, 206, 205, 196, 205, 201, 203, 203, 204, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.414963366Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135852024-02-04T14:06:17.420Z{"msg":"[0]R generation[0..12]: [123, 121, 113, 125, 124, 120, 124, 121, 122, 122, 123, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415015093Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135862024-02-04T14:06:17.420Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415048408Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135872024-02-04T14:06:17.420Z{"msg":"[1]R flush_numbers[0..12]: [204, 201, 185, 206, 205, 196, 205, 201, 203, 203, 204, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415072641Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135882024-02-04T14:06:17.420ZWait for a query_work_queue command to finish before sending IO
135892024-02-04T14:06:17.420Z{"msg":"[1]R generation[0..12]: [123, 121, 113, 125, 124, 120, 124, 121, 122, 122, 123, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415129322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135902024-02-04T14:06:17.421Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415156053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135912024-02-04T14:06:17.421Z{"msg":"[2]R flush_numbers[0..12]: [204, 201, 185, 206, 205, 196, 205, 201, 203, 203, 204, 206]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415205558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135922024-02-04T14:06:17.421Z{"msg":"[2]R generation[0..12]: [123, 121, 113, 125, 124, 120, 124, 121, 122, 122, 123, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415230035Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135932024-02-04T14:06:17.421Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.4152788Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135942024-02-04T14:06:17.421Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.41530193Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135952024-02-04T14:06:17.421Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415326049Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135962024-02-04T14:06:17.422Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415355475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135972024-02-04T14:06:17.422Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415378237Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135982024-02-04T14:06:17.422Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415400516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
135992024-02-04T14:06:17.422Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415420838Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
136002024-02-04T14:06:17.422Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415447621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
136012024-02-04T14:06:17.422Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415491901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
136022024-02-04T14:06:17.422Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415515794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
136032024-02-04T14:06:17.423Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415568789Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
136042024-02-04T14:06:17.423Z{"msg":"79fcdc01-79ac-4254-9be1-d07a55ce319f is now active with session: cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.41559164Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
136052024-02-04T14:06:17.423Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415614565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633,"session_id":"cfe96fa5-7106-4d01-b1d3-fbb4c5d89f28"}
136062024-02-04T14:06:17.423Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:16.415667763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633}
136072024-02-04T14:06:17.423ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
136082024-02-04T14:06:17.423ZLoading write count information from file "/tmp/test_repair_verify.data"
136092024-02-04T14:06:17.424ZRun Repair workload
136102024-02-04T14:06:17.424Z01/30 Write block 460 len 28672 data: 9 8 7 9 8 10 12
136112024-02-04T14:06:17.424Z02/30 Flush
136122024-02-04T14:06:17.442Z03/30 Flush
136132024-02-04T14:06:17.442Z04/30 Write block 340 len 32768 data: 9 8 7 8 9 10 9 10
136142024-02-04T14:06:17.442Z05/30 Read block 532 len 32768
136152024-02-04T14:06:17.442Z06/30 Read block 279 len 28672
136162024-02-04T14:06:17.442Z07/30 Write block 119 len 12288 data: 10 9 10
136172024-02-04T14:06:17.442Z08/30 Write block 139 len 8192 data: 7 11
136182024-02-04T14:06:17.442Z09/30 Write block 289 len 4096 data: 13
136192024-02-04T14:06:17.442Z10/30 Read block 305 len 36864
136202024-02-04T14:06:17.442Z11/30 Read block 550 len 12288
136212024-02-04T14:06:17.443Z12/30 Read block 175 len 40960
136222024-02-04T14:06:17.443Z13/30 Read block 144 len 4096
136232024-02-04T14:06:17.443Z14/30 Read block 210 len 4096
136242024-02-04T14:06:17.443Z15/30 Write block 240 len 36864 data: 12 12 9 8 9 8 10 11 15
136252024-02-04T14:06:17.443Z16/30 Read block 499 len 12288
136262024-02-04T14:06:17.443Z17/30 Write block 567 len 28672 data: 11 10 8 9 8 8 8
136272024-02-04T14:06:17.443Z18/30 Write block 46 len 20480 data: 9 7 8 5 4
136282024-02-04T14:06:17.443Z19/30 Read block 138 len 36864
136292024-02-04T14:06:17.444Z20/30 Read block 89 len 4096
136302024-02-04T14:06:17.444Z21/30 Write block 334 len 12288 data: 8 7 7
136312024-02-04T14:06:17.444Z22/30 Write block 425 len 12288 data: 7 7 6
136322024-02-04T14:06:17.444Z23/30 Read block 233 len 16384
136332024-02-04T14:06:17.444Z24/30 Write block 566 len 24576 data: 13 12 11 9 10 9
136342024-02-04T14:06:17.444Z25/30 Write block 48 len 24576 data: 9 6 5 6 6 6
136352024-02-04T14:06:17.444Z26/30 Write block 419 len 40960 data: 10 13 11 10 10 6 8 8 7 7
136362024-02-04T14:06:17.445Z27/30 Write block 412 len 20480 data: 11 11 12 10 8
136372024-02-04T14:06:17.445Z28/30 Write block 237 len 16384 data: 13 11 10 13
136382024-02-04T14:06:17.445Z29/30 Write block 54 len 28672 data: 6 7 9 10 8 10 12
136392024-02-04T14:06:17.445Z30/30 Write block 46 len 8192 data: 10 8
136402024-02-04T14:06:17.446Z----------------------------------------------------------------
136412024-02-04T14:06:17.446Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
136422024-02-04T14:06:17.446ZStates: Active Active Active
136432024-02-04T14:06:17.446ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136442024-02-04T14:06:17.446Z 1 Acked 1000 Write 7 Done Done Sent false
136452024-02-04T14:06:17.446Z 2 Acked 1001 Flush 0 Done Done Sent false
136462024-02-04T14:06:17.446Z 3 Acked 1002 Flush 0 Done Done Sent false
136472024-02-04T14:06:17.446Z 4 Acked 1003 Write 8 Done Done Sent false
136482024-02-04T14:06:17.447Z 5 Acked 1004 Read 8 Done Done Sent false
136492024-02-04T14:06:17.447Z 6 Acked 1005 Read 7 Done Done Sent false
136502024-02-04T14:06:17.447Z 7 Acked 1006 Write 3 Done Done Sent false
136512024-02-04T14:06:17.447Z 8 Acked 1007 Write 2 Done Done Sent false
136522024-02-04T14:06:17.447Z 9 Acked 1008 Write 1 Done Done Sent false
136532024-02-04T14:06:17.447Z 10 Acked 1009 Read 9 Done Done Sent false
136542024-02-04T14:06:17.447Z 11 Acked 1010 Read 3 Done Done Sent false
136552024-02-04T14:06:17.448Z 12 Acked 1011 Read 10 Done Done Sent false
136562024-02-04T14:06:17.448Z 13 Acked 1012 Read 1 Done Done Sent false
136572024-02-04T14:06:17.448Z 14 Acked 1013 Read 1 Done Done Sent false
136582024-02-04T14:06:17.448Z 15 Acked 1014 Write 9 Done Done Sent false
136592024-02-04T14:06:17.448Z 16 Acked 1015 Read 3 Done Done Sent false
136602024-02-04T14:06:17.449Z 17 Acked 1016 Write 7 Done Done Sent false
136612024-02-04T14:06:17.449Z 18 Acked 1017 Write 5 Done Done Sent false
136622024-02-04T14:06:17.449Z 19 Acked 1018 Read 9 Done Done Sent false
136632024-02-04T14:06:17.449Z 20 Acked 1019 Read 1 Done Done Sent false
136642024-02-04T14:06:17.449Z 21 Acked 1020 Write 3 Done Done Sent false
136652024-02-04T14:06:17.450Z 22 Acked 1021 Write 3 Done Done Sent false
136662024-02-04T14:06:17.450Z 23 Acked 1022 Read 4 Done Done Sent false
136672024-02-04T14:06:17.450Z 24 Acked 1023 Write 6 Done Done Sent false
136682024-02-04T14:06:17.450Z 25 Acked 1024 Write 6 Done Done Sent false
136692024-02-04T14:06:17.450Z 26 Acked 1025 Write 10 Done Done Sent false
136702024-02-04T14:06:17.451Z 27 Acked 1026 Write 5 Done Done Sent false
136712024-02-04T14:06:17.451Z 28 Acked 1027 Write 4 Done Done Sent false
136722024-02-04T14:06:17.451Z 29 Acked 1028 Write 7 Sent Sent Sent false
136732024-02-04T14:06:17.451Z 30 Acked 1029 Write 2 Sent Sent Sent false
136742024-02-04T14:06:17.451Z STATES DS:0 DS:1 DS:2 TOTAL
136752024-02-04T14:06:17.452Z New 0 0 0 0
136762024-02-04T14:06:17.452Z Sent 2 2 30 34
136772024-02-04T14:06:17.452Z Done 28 28 0 56
136782024-02-04T14:06:17.452Z Skipped 0 0 0 0
136792024-02-04T14:06:17.453Z Error 0 0 0 0
136802024-02-04T14:06:17.453ZLast Flush: 1002 1002 0
136812024-02-04T14:06:17.453ZDownstairs last five completed:
136822024-02-04T14:06:17.453ZUpstairs last five completed: 30 29 28 27 26
136832024-02-04T14:06:17.453Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:16.458635558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1633}
136842024-02-04T14:06:17.453ZWrote out file "/tmp/test_repair_verify.data"
136852024-02-04T14:06:17.467Z
136862024-02-04T14:06:17.467ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136872024-02-04T14:06:17.471ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136882024-02-04T14:06:17.471Z 0 000-019 123 123 123 204 204 204 F F F
136892024-02-04T14:06:17.471Z 1 020-039 121 121 121 201 201 201 F F F
136902024-02-04T14:06:17.472Z 2 040-059 113 113 113 185 185 185 T T F <---
136912024-02-04T14:06:17.472Z 3 060-079 125 125 125 206 206 206 T T F <---
136922024-02-04T14:06:17.472Z 4 080-099 124 124 124 205 205 205 F F F
136932024-02-04T14:06:17.472Z 5 100-119 120 120 120 196 196 196 T T F <---
136942024-02-04T14:06:17.472Z 6 120-139 124 124 124 205 205 205 T T F <---
136952024-02-04T14:06:17.472Z 7 140-159 121 121 121 201 201 201 T T F <---
136962024-02-04T14:06:17.472Z 8 160-179 122 122 122 203 203 203 F F F
136972024-02-04T14:06:17.472Z 9 180-199 122 122 122 203 203 203 F F F
136982024-02-04T14:06:17.473Z 10 200-219 123 123 123 204 204 204 F F F
136992024-02-04T14:06:17.473Z 11 220-239 125 125 125 206 206 206 T T F <---
137002024-02-04T14:06:17.473Z 12 240-259 120 120 120 196 196 196 T T F <---
137012024-02-04T14:06:17.473Z 13 260-279 124 124 124 205 205 205 F F F
137022024-02-04T14:06:17.473Z 14 280-299 123 123 123 204 204 204 T T F <---
137032024-02-04T14:06:17.473Z 15 300-319 120 120 120 200 200 200 F F F
137042024-02-04T14:06:17.473Z 16 320-339 116 116 116 189 189 189 T T F <---
137052024-02-04T14:06:17.473Z 17 340-359 124 124 124 205 205 205 T T F <---
137062024-02-04T14:06:17.473Z 18 360-379 116 116 116 188 188 188 F F F
137072024-02-04T14:06:17.473Z 19 380-399 125 125 125 206 206 206 F F F
137082024-02-04T14:06:17.474Z 20 400-419 117 117 117 192 192 192 T T F <---
137092024-02-04T14:06:17.474Z 21 420-439 117 117 117 192 192 192 T T F <---
137102024-02-04T14:06:17.474Z 22 440-459 125 125 125 206 206 206 F F F
137112024-02-04T14:06:17.474Z 23 460-479 126 126 116 207 207 191 F F F <---
137122024-02-04T14:06:17.474Z 24 480-499 124 124 124 205 205 205 F F F
137132024-02-04T14:06:17.474Z 25 500-519 125 125 125 206 206 206 F F F
137142024-02-04T14:06:17.474Z 26 520-539 125 125 125 206 206 206 F F F
137152024-02-04T14:06:17.474Z 27 540-559 125 125 125 206 206 206 F F F
137162024-02-04T14:06:17.475Z 28 560-579 125 125 125 206 206 206 T T F <---
137172024-02-04T14:06:17.475Z 29 580-599 124 124 124 205 205 205 F F F
137182024-02-04T14:06:17.475ZMax gen: 126, Max flush: 207
137192024-02-04T14:06:17.475ZError: Difference in extent metadata found!
137202024-02-04T14:06:17.479ZA Difference in extent metadata is expected here
137212024-02-04T14:06:17.479ZOn loop 63
137222024-02-04T14:06:17.479Z
137232024-02-04T14:06:17.479ZVerifying data now
137242024-02-04T14:06:22.704ZNo change after verify
137252024-02-04T14:06:22.705ZLoop: 63 Downstairs dump after verify (and repair):
137262024-02-04T14:06:22.728ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137272024-02-04T14:06:22.728Z 0 000-019 123 123 123 204 204 204 F F F
137282024-02-04T14:06:22.728Z 1 020-039 121 121 121 201 201 201 F F F
137292024-02-04T14:06:22.728Z 2 040-059 127 127 127 208 208 208 F F F
137302024-02-04T14:06:22.728Z 3 060-079 127 127 127 208 208 208 F F F
137312024-02-04T14:06:22.728Z 4 080-099 124 124 124 205 205 205 F F F
137322024-02-04T14:06:22.728Z 5 100-119 127 127 127 208 208 208 F F F
137332024-02-04T14:06:22.728Z 6 120-139 127 127 127 208 208 208 F F F
137342024-02-04T14:06:22.728Z 7 140-159 127 127 127 208 208 208 F F F
137352024-02-04T14:06:22.728Z 8 160-179 122 122 122 203 203 203 F F F
137362024-02-04T14:06:22.731Z 9 180-199 122 122 122 203 203 203 F F F
137372024-02-04T14:06:22.731Z 10 200-219 123 123 123 204 204 204 F F F
137382024-02-04T14:06:22.732Z 11 220-239 127 127 127 208 208 208 F F F
137392024-02-04T14:06:22.732Z 12 240-259 127 127 127 208 208 208 F F F
137402024-02-04T14:06:22.732Z 13 260-279 124 124 124 205 205 205 F F F
137412024-02-04T14:06:22.732Z 14 280-299 127 127 127 208 208 208 F F F
137422024-02-04T14:06:22.732Z 15 300-319 120 120 120 200 200 200 F F F
137432024-02-04T14:06:22.732Z 16 320-339 127 127 127 208 208 208 F F F
137442024-02-04T14:06:22.732Z 17 340-359 127 127 127 208 208 208 F F F
137452024-02-04T14:06:22.732Z 18 360-379 116 116 116 188 188 188 F F F
137462024-02-04T14:06:22.732Z 19 380-399 125 125 125 206 206 206 F F F
137472024-02-04T14:06:22.732Z 20 400-419 127 127 127 208 208 208 F F F
137482024-02-04T14:06:22.732Z 21 420-439 127 127 127 208 208 208 F F F
137492024-02-04T14:06:22.732Z 22 440-459 125 125 125 206 206 206 F F F
137502024-02-04T14:06:22.732Z 23 460-479 126 126 126 207 207 207 F F F
137512024-02-04T14:06:22.732Z 24 480-499 124 124 124 205 205 205 F F F
137522024-02-04T14:06:22.732Z 25 500-519 125 125 125 206 206 206 F F F
137532024-02-04T14:06:22.732Z 26 520-539 125 125 125 206 206 206 F F F
137542024-02-04T14:06:22.732Z 27 540-559 125 125 125 206 206 206 F F F
137552024-02-04T14:06:22.732Z 28 560-579 127 127 127 208 208 208 F F F
137562024-02-04T14:06:22.732Z 29 580-599 124 124 124 205 205 205 F F F
137572024-02-04T14:06:22.732ZMax gen: 127, Max flush: 208
137582024-02-04T14:06:22.736Z
137592024-02-04T14:06:22.736ZBegin loop 64
137602024-02-04T14:06:22.736ZDownstairs to restart: 1
137612024-02-04T14:06:22.765Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.797631236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641}
137622024-02-04T14:06:22.765Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.79818378Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641}
137632024-02-04T14:06:22.769Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.798238828Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641}
137642024-02-04T14:06:22.769Z{"msg":"Crucible c61b9503-d405-4a08-88b4-3e5c2e4dd61e has session id: eb107ecf-1b13-45d2-82bc-a1e455d34d0e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.798291257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137652024-02-04T14:06:22.769Z{"msg":"Upstairs opts: Upstairs UUID: c61b9503-d405-4a08-88b4-3e5c2e4dd61e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.798323339Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137662024-02-04T14:06:22.769Z{"msg":"Crucible stats registered with UUID: c61b9503-d405-4a08-88b4-3e5c2e4dd61e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.79837954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137672024-02-04T14:06:22.769ZCrucible runtime is spawned
137682024-02-04T14:06:22.769Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.798788609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641}
137692024-02-04T14:06:22.769Z{"msg":"c61b9503-d405-4a08-88b4-3e5c2e4dd61e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799120958Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137702024-02-04T14:06:22.769Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799228971Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137712024-02-04T14:06:22.769Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799278425Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137722024-02-04T14:06:22.769Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799319789Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137732024-02-04T14:06:22.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799359821Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137742024-02-04T14:06:22.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799395071Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137752024-02-04T14:06:22.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799434373Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137762024-02-04T14:06:22.770Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799474191Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"io task","client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137772024-02-04T14:06:22.770Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799514528Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"io task","client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137782024-02-04T14:06:22.770Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799552492Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"io task","client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137792024-02-04T14:06:22.770Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799577141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"io task","client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137802024-02-04T14:06:22.770Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799603933Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"io task","client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137812024-02-04T14:06:22.770Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.799781271Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"io task","client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137822024-02-04T14:06:22.770Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.800800863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137832024-02-04T14:06:22.770Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.800871805Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137842024-02-04T14:06:22.770Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.800915837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137852024-02-04T14:06:22.770Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.800986094Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137862024-02-04T14:06:22.770Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.801751176Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137872024-02-04T14:06:22.774Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.802020461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137882024-02-04T14:06:22.774Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.802667121Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137892024-02-04T14:06:22.774Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.802744389Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137902024-02-04T14:06:22.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.802789508Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137912024-02-04T14:06:22.775Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.802821834Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137922024-02-04T14:06:22.775Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.80294085Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137932024-02-04T14:06:22.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.803054693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137942024-02-04T14:06:22.775Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.803120247Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137952024-02-04T14:06:22.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.803165121Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137962024-02-04T14:06:22.775Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.803925178Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137972024-02-04T14:06:22.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.804664639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137982024-02-04T14:06:22.775Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.804716944Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
137992024-02-04T14:06:22.775Z{"msg":"[0]R flush_numbers[0..12]: [204, 201, 208, 208, 205, 208, 208, 208, 203, 203, 204, 208]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.804743825Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138002024-02-04T14:06:22.775Z{"msg":"[0]R generation[0..12]: [123, 121, 127, 127, 124, 127, 127, 127, 122, 122, 123, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.804767637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138012024-02-04T14:06:22.775Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.80479202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138022024-02-04T14:06:22.775Z{"msg":"[1]R flush_numbers[0..12]: [204, 201, 208, 208, 205, 208, 208, 208, 203, 203, 204, 208]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.804835294Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138032024-02-04T14:06:22.775Z{"msg":"[1]R generation[0..12]: [123, 121, 127, 127, 124, 127, 127, 127, 122, 122, 123, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.804859935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138042024-02-04T14:06:22.775Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.804902156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138052024-02-04T14:06:22.775Z{"msg":"[2]R flush_numbers[0..12]: [204, 201, 208, 208, 205, 208, 208, 208, 203, 203, 204, 208]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.80492666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138062024-02-04T14:06:22.775Z{"msg":"[2]R generation[0..12]: [123, 121, 127, 127, 124, 127, 127, 127, 122, 122, 123, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.804968577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"Wait for a query_work_queue command to finish before sending IO
138072024-02-04T14:06:22.775Z}
138082024-02-04T14:06:22.775Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.80499734Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138092024-02-04T14:06:22.775Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.80503782Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138102024-02-04T14:06:22.775Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805062598Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138112024-02-04T14:06:22.775Z{"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805126091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138122024-02-04T14:06:22.775Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805153931Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138132024-02-04T14:06:22.776Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805212104Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138142024-02-04T14:06:22.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805248265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138152024-02-04T14:06:22.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.80531632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138162024-02-04T14:06:22.776Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805351995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138172024-02-04T14:06:22.776Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805410944Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138182024-02-04T14:06:22.776Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805480542Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138192024-02-04T14:06:22.776Z{"msg":"c61b9503-d405-4a08-88b4-3e5c2e4dd61e is now active with session: eb107ecf-1b13-45d2-82bc-a1e455d34d0e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.80551857Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"eb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138202024-02-04T14:06:22.776Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.805578676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138212024-02-04T14:06:22.776Zeb107ecf-1b13-45d2-82bc-a1e455d34d0e"}
138222024-02-04T14:06:22.776Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:21.80570202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641}
138232024-02-04T14:06:22.776ZLoading write count information from file "/tmp/test_repair_verify.data"
138242024-02-04T14:06:22.776ZRun Repair workload
138252024-02-04T14:06:22.776Z01/30 Write block 33 len 16384 data: 9 9 12 13
138262024-02-04T14:06:22.779Z02/30 Read block 68 len 8192
138272024-02-04T14:06:22.779Z03/30 Read block 375 len 32768
138282024-02-04T14:06:22.779Z04/30 Flush
138292024-02-04T14:06:22.782Z05/30 Write block 190 len 20480 data: 8 9 9 9 7
138302024-02-04T14:06:22.786Z06/30 Read block 156 len 28672
138312024-02-04T14:06:22.786Z07/30 Write block 407 len 12288 data: 12 11 12
138322024-02-04T14:06:22.786Z08/30 Read block 367 len 28672
138332024-02-04T14:06:22.789Z09/30 Read block 482 len 40960
138342024-02-04T14:06:22.793Z10/30 Read block 322 len 16384
138352024-02-04T14:06:22.793Z11/30 Flush
138362024-02-04T14:06:22.797Z12/30 Write block 348 len 8192 data: 11 14
138372024-02-04T14:06:22.797Z13/30 Flush
138382024-02-04T14:06:22.798Z14/30 Write block 219 len 16384 data: 8 10 11 8
138392024-02-04T14:06:22.805Z15/30 Write block 77 len 40960 data: 12 11 10 12 13 14 12 12 12 11
138402024-02-04T14:06:22.805Z16/30 Read block 343 len 32768
138412024-02-04T14:06:22.805Z17/30 Read block 75 len 12288
138422024-02-04T14:06:22.805Z18/30 Read block 543 len 4096
138432024-02-04T14:06:23.215Z19/30 Read block 507 len 24576
138442024-02-04T14:06:23.216Z20/30 Write block 483 len 12288 data: 5 7 10
138452024-02-04T14:06:23.216Z21/30 Write block 391 len 12288 data: 11 12 14
138462024-02-04T14:06:23.216Z22/30 Read block 138 len 16384
138472024-02-04T14:06:23.216Z23/30 Write block 499 len 4096 data: 14
138482024-02-04T14:06:23.216Z24/30 Read block 482 len 40960
138492024-02-04T14:06:23.232Z25/30 Write block 503 len 16384 data: 11 13 10 14
138502024-02-04T14:06:23.232Z26/30 Flush
138512024-02-04T14:06:23.233Z27/30 Write block 134 len 32768 data: 7 8 6 5 6 8 12 8
138522024-02-04T14:06:23.233Z28/30 Read block 436 len 20480
138532024-02-04T14:06:23.233Z29/30 Write block 120 len 20480 data: 10 11 10 9 8
138542024-02-04T14:06:23.233Z30/30 Read block 217 len 32768
138552024-02-04T14:06:23.233Z----------------------------------------------------------------
138562024-02-04T14:06:23.233Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:26
138572024-02-04T14:06:23.233ZStates: Active Active Active
138582024-02-04T14:06:23.233ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138592024-02-04T14:06:23.233Z 5 Acked 1004 Write 5 Done Done Done false
138602024-02-04T14:06:23.233Z 6 Acked 1005 Read 7 Done Done Done false
138612024-02-04T14:06:23.233Z 7 Acked 1006 Write 3 Done Done Done false
138622024-02-04T14:06:23.233Z 8 Acked 1007 Read 7 Done Sent Done false
138632024-02-04T14:06:23.234Z 9 Acked 1008 Read 10 Done Sent Done false
138642024-02-04T14:06:23.234Z 10 Acked 1009 Read 4 Done Sent Done false
138652024-02-04T14:06:23.234Z 11 Acked 1010 Flush 0 Done Sent Done false
138662024-02-04T14:06:23.234Z 12 Acked 1011 Write 2 Done Sent Done false
138672024-02-04T14:06:23.234Z 13 Acked 1012 Flush 0 Done Sent Done false
138682024-02-04T14:06:23.234Z 14 Acked 1013 Write 4 Done Sent Done false
138692024-02-04T14:06:23.234Z 15 Acked 1014 Write 10 Done Sent Done false
138702024-02-04T14:06:23.234Z 16 Acked 1015 Read 8 Done Sent Done false
138712024-02-04T14:06:23.234Z 17 Acked 1016 Read 3 Done Sent Done false
138722024-02-04T14:06:23.234Z 18 Acked 1017 Read 1 Done Sent Done false
138732024-02-04T14:06:23.235Z 19 Acked 1018 Read 6 Done Sent Done false
138742024-02-04T14:06:23.235Z 20 Acked 1019 Write 3 Done Sent Done false
138752024-02-04T14:06:23.235Z 21 Acked 1020 Write 3 Done Sent Done false
138762024-02-04T14:06:23.235Z 22 Acked 1021 Read 4 Done Sent Done false
138772024-02-04T14:06:23.235Z 23 Acked 1022 Write 1 Done Sent Done false
138782024-02-04T14:06:23.235Z 24 Acked 1023 Read 10 Done Sent Done false
138792024-02-04T14:06:23.235Z 25 Acked 1024 Write 4 Done Sent Done false
138802024-02-04T14:06:23.235Z 26 Acked 1025 Flush 0 Done Sent Done false
138812024-02-04T14:06:23.235Z 27 Acked 1026 Write 8 Done Sent Done false
138822024-02-04T14:06:23.235Z 28 Acked 1027 Read 5 Done Sent Done false
138832024-02-04T14:06:23.236Z 29 Acked 1028 Write 5 Done Sent Done false
138842024-02-04T14:06:23.236Z 30 Acked 1029 Read 8 Done Sent Done false
138852024-02-04T14:06:23.236Z STATES DS:0 DS:1 DS:2 TOTAL
138862024-02-04T14:06:23.236Z New 0 0 0 0
138872024-02-04T14:06:23.236Z Sent 0 23 0 23
138882024-02-04T14:06:23.236Z Done 26 3 26 55
138892024-02-04T14:06:23.236Z Skipped 0 0 0 0
138902024-02-04T14:06:23.236Z Error 0 0 0 0
138912024-02-04T14:06:23.236ZLast Flush: 1025 1003 1025
138922024-02-04T14:06:23.236ZDownstairs last five completed: 1003 1002 1001 1000
138932024-02-04T14:06:23.237ZUpstairs last five completed: 30 29 28 27 26
138942024-02-04T14:06:23.237Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:21.855100404Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1641}
138952024-02-04T14:06:23.237ZWrote out file "/tmp/test_repair_verify.data"
138962024-02-04T14:06:23.237Z
138972024-02-04T14:06:23.237ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138982024-02-04T14:06:23.237ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138992024-02-04T14:06:23.237Z 0 000-019 123 123 123 204 204 204 F F F
139002024-02-04T14:06:23.237Z 1 020-039 128 128 128 209 209 209 F F F
139012024-02-04T14:06:23.238Z 2 040-059 127 127 127 208 208 208 F F F
139022024-02-04T14:06:23.238Z 3 060-079 128 127 128 212 208 212 F F F <---
139032024-02-04T14:06:23.238Z 4 080-099 128 124 128 212 205 212 F F F <---
139042024-02-04T14:06:23.238Z 5 100-119 127 127 127 208 208 208 F F F
139052024-02-04T14:06:23.238Z 6 120-139 127 127 127 208 208 208 T F T <---
139062024-02-04T14:06:23.238Z 7 140-159 127 127 127 208 208 208 T F T <---
139072024-02-04T14:06:23.238Z 8 160-179 122 122 122 203 203 203 F F F
139082024-02-04T14:06:23.238Z 9 180-199 128 122 128 210 203 210 F T F <---
139092024-02-04T14:06:23.238Z 10 200-219 128 123 128 212 204 212 F F F <---
139102024-02-04T14:06:23.238Z 11 220-239 128 127 128 212 208 212 F F F <---
139112024-02-04T14:06:23.239Z 12 240-259 127 127 127 208 208 208 F F F
139122024-02-04T14:06:23.239Z 13 260-279 124 124 124 205 205 205 F F F
139132024-02-04T14:06:23.239Z 14 280-299 127 127 127 208 208 208 F F F
139142024-02-04T14:06:23.239Z 15 300-319 120 120 120 200 200 200 F F F
139152024-02-04T14:06:23.239Z 16 320-339 127 127 127 208 208 208 F F F
139162024-02-04T14:06:23.239Z 17 340-359 128 127 128 211 208 211 F F F <---
139172024-02-04T14:06:23.239Z 18 360-379 116 116 116 188 188 188 F F F
139182024-02-04T14:06:23.239Z 19 380-399 128 125 128 212 206 212 F F F <---
139192024-02-04T14:06:23.239Z 20 400-419 128 127 128 210 208 210 F T F <---
139202024-02-04T14:06:23.239Z 21 420-439 127 127 127 208 208 208 F F F
139212024-02-04T14:06:23.240Z 22 440-459 125 125 125 206 206 206 F F F
139222024-02-04T14:06:23.240Z 23 460-479 126 126 126 207 207 207 F F F
139232024-02-04T14:06:23.240Z 24 480-499 128 124 128 212 205 212 F F F <---
139242024-02-04T14:06:23.240Z 25 500-519 128 125 128 212 206 212 F F F <---
139252024-02-04T14:06:23.240Z 26 520-539 125 125 125 206 206 206 F F F
139262024-02-04T14:06:23.240Z 27 540-559 125 125 125 206 206 206 F F F
139272024-02-04T14:06:23.240Z 28 560-579 127 127 127 208 208 208 F F F
139282024-02-04T14:06:23.240Z 29 580-599 124 124 124 205 205 205 F F F
139292024-02-04T14:06:23.240ZMax gen: 128, Max flush: 212
139302024-02-04T14:06:23.240ZError: Difference in extent metadata found!
139312024-02-04T14:06:23.240ZA Difference in extent metadata is expected here
139322024-02-04T14:06:23.241ZOn loop 64
139332024-02-04T14:06:23.241Z
139342024-02-04T14:06:23.241ZVerifying data now
139352024-02-04T14:06:28.958ZNo change after verify
139362024-02-04T14:06:28.958ZLoop: 64 Downstairs dump after verify (and repair):
139372024-02-04T14:06:28.981ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139382024-02-04T14:06:28.981Z 0 000-019 123 123 123 204 204 204 F F F
139392024-02-04T14:06:28.982Z 1 020-039 128 128 128 209 209 209 F F F
139402024-02-04T14:06:28.982Z 2 040-059 127 127 127 208 208 208 F F F
139412024-02-04T14:06:28.982Z 3 060-079 128 128 128 212 212 212 F F F
139422024-02-04T14:06:28.982Z 4 080-099 128 128 128 212 212 212 F F F
139432024-02-04T14:06:28.982Z 5 100-119 127 127 127 208 208 208 F F F
139442024-02-04T14:06:28.982Z 6 120-139 129 129 129 213 213 213 F F F
139452024-02-04T14:06:28.982Z 7 140-159 129 129 129 213 213 213 F F F
139462024-02-04T14:06:28.982Z 8 160-179 122 122 122 203 203 203 F F F
139472024-02-04T14:06:28.982Z 9 180-199 128 128 128 210 210 210 F F F
139482024-02-04T14:06:28.985Z 10 200-219 128 128 128 212 212 212 F F F
139492024-02-04T14:06:28.985Z 11 220-239 128 128 128 212 212 212 F F F
139502024-02-04T14:06:28.985Z 12 240-259 127 127 127 208 208 208 F F F
139512024-02-04T14:06:28.985Z 13 260-279 124 124 124 205 205 205 F F F
139522024-02-04T14:06:28.985Z 14 280-299 127 127 127 208 208 208 F F F
139532024-02-04T14:06:28.985Z 15 300-319 120 120 120 200 200 200 F F F
139542024-02-04T14:06:28.985Z 16 320-339 127 127 127 208 208 208 F F F
139552024-02-04T14:06:28.985Z 17 340-359 128 128 128 211 211 211 F F F
139562024-02-04T14:06:28.985Z 18 360-379 116 116 116 188 188 188 F F F
139572024-02-04T14:06:28.986Z 19 380-399 128 128 128 212 212 212 F F F
139582024-02-04T14:06:28.986Z 20 400-419 128 128 128 210 210 210 F F F
139592024-02-04T14:06:28.986Z 21 420-439 127 127 127 208 208 208 F F F
139602024-02-04T14:06:28.986Z 22 440-459 125 125 125 206 206 206 F F F
139612024-02-04T14:06:28.986Z 23 460-479 126 126 126 207 207 207 F F F
139622024-02-04T14:06:28.986Z 24 480-499 128 128 128 212 212 212 F F F
139632024-02-04T14:06:28.986Z 25 500-519 128 128 128 212 212 212 F F F
139642024-02-04T14:06:28.986Z 26 520-539 125 125 125 206 206 206 F F F
139652024-02-04T14:06:28.986Z 27 540-559 125 125 125 206 206 206 F F F
139662024-02-04T14:06:28.986Z 28 560-579 127 127 127 208 208 208 F F F
139672024-02-04T14:06:28.986Z 29 580-599 124 124 124 205 205 205 F F F
139682024-02-04T14:06:28.986ZMax gen: 129, Max flush: 213
139692024-02-04T14:06:28.989Z
139702024-02-04T14:06:28.989ZBegin loop 65
139712024-02-04T14:06:28.989ZDownstairs to restart: 1
139722024-02-04T14:06:29.044Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.051848646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649}
139732024-02-04T14:06:29.044Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052410175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal"Crucible runtime is spawned
139742024-02-04T14:06:29.044Z,"pid":1649}
139752024-02-04T14:06:29.051Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052489267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649}
139762024-02-04T14:06:29.051Z{"msg":"Crucible 929c8e72-4610-46e4-9f43-775729133990 has session id: b8c9ffdb-3a9c-4e62-a77b-27dd0292323b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052573613Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139772024-02-04T14:06:29.051Z{"msg":"Upstairs opts: Upstairs UUID: 929c8e72-4610-46e4-9f43-775729133990, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052611801Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139782024-02-04T14:06:29.051Z{"msg":"Crucible stats registered with UUID: 929c8e72-4610-46e4-9f43-775729133990","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052641506Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139792024-02-04T14:06:29.051Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052700228Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649}
139802024-02-04T14:06:29.051Z{"msg":"929c8e72-4610-46e4-9f43-775729133990 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052881193Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139812024-02-04T14:06:29.051Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052939911Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139822024-02-04T14:06:29.051Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052966131Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139832024-02-04T14:06:29.051Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.052989176Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139842024-02-04T14:06:29.051Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.05303866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139852024-02-04T14:06:29.051Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.053070574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139862024-02-04T14:06:29.051Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.053094367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139872024-02-04T14:06:29.051Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.053117281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139882024-02-04T14:06:29.051Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.053173699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139892024-02-04T14:06:29.051Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.053266919Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139902024-02-04T14:06:29.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.053409211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139912024-02-04T14:06:29.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.053465853Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139922024-02-04T14:06:29.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.053514249Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139932024-02-04T14:06:29.051Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.054684612Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139942024-02-04T14:06:29.052Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.054781515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139952024-02-04T14:06:29.052Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.054878855Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139962024-02-04T14:06:29.052Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.054931168Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139972024-02-04T14:06:29.052Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.055813629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139982024-02-04T14:06:29.052Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.055894271Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
139992024-02-04T14:06:29.052Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.056004369Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140002024-02-04T14:06:29.052Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.056088804Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140012024-02-04T14:06:29.052Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.057095886Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140022024-02-04T14:06:29.052Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.057159617Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140032024-02-04T14:06:29.052Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.057185662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140042024-02-04T14:06:29.052Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.057208352Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140052024-02-04T14:06:29.052Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.057373202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140062024-02-04T14:06:29.052Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.05742631Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140072024-02-04T14:06:29.052Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.057451555Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140082024-02-04T14:06:29.052Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058170771Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140092024-02-04T14:06:29.052Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058233595Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140102024-02-04T14:06:29.052Z{"msg":"[0]R flush_numbers[0..12]: [204, 209, 208, 212, 212, 208, 213, 213, 203, 210, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058261261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140112024-02-04T14:06:29.052Z{"msg":"[0]R generation[0..12]: [123, 128, 127, 128, 128, 127, 129, 129, 122, 128, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058304574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140122024-02-04T14:06:29.052Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058330291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140132024-02-04T14:06:29.052Z{"msg":"[1]R flush_numbers[0..12]: [204, 209, 208, 212, 212, 208, 213, 213, 203, 210, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058372752Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140142024-02-04T14:06:29.052Z{"msg":"[1]R generation[0..12]: [123, 128, 127, 128, 128, 127, 129, 129, 122, 128, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058397627Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140152024-02-04T14:06:29.052Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058440826Z","hostname"Wait for a query_work_queue command to finish before sending IO
140162024-02-04T14:06:29.052Z:"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140172024-02-04T14:06:29.053Z{"msg":"[2]R flush_numbers[0..12]: [204, 209, 208, 212, 212, 208, 213, 213, 203, 210, 212, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058469896Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140182024-02-04T14:06:29.053Z{"msg":"[2]R generation[0..12]: [123, 128, 127, 128, 128, 127, 129, 129, 122, 128, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058519625Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140192024-02-04T14:06:29.053Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.05854457Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140202024-02-04T14:06:29.053Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058585374Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140212024-02-04T14:06:29.053Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058608661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140222024-02-04T14:06:29.053Z{"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058630116Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140232024-02-04T14:06:29.053Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058669985Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140242024-02-04T14:06:29.053Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058694247Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140252024-02-04T14:06:29.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058736114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140262024-02-04T14:06:29.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058759067Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140272024-02-04T14:06:29.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058805855Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140282024-02-04T14:06:29.053Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058829504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140292024-02-04T14:06:29.053Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058850215Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140302024-02-04T14:06:29.053Z{"msg":"929c8e72-4610-46e4-9f43-775729133990 is now active with session: b8c9ffdb-3a9c-4e62-a77b-27dd0292323b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058894428Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140312024-02-04T14:06:29.053Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058915727Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649,"session_id":"b8c9ffdb-3a9c-4e62-a77b-27dd0292323b"}
140322024-02-04T14:06:29.053Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:28.058937449Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649}
140332024-02-04T14:06:29.053ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140342024-02-04T14:06:29.053ZLoading write count information from file "/tmp/test_repair_verify.data"
140352024-02-04T14:06:29.053ZRun Repair workload
140362024-02-04T14:06:29.053Z01/30 Write block 355 len 40960 data: 14 15 10 9 8 8 9 10 10 9
140372024-02-04T14:06:29.053Z02/30 Read block 120 len 36864
140382024-02-04T14:06:29.053Z03/30 Read block 483 len 24576
140392024-02-04T14:06:29.053Z04/30 Write block 402 len 24576 data: 11 10 10 13 12 13
140402024-02-04T14:06:29.053Z05/30 Write block 5 len 12288 data: 8 9 9
140412024-02-04T14:06:29.053Z06/30 Read block 60 len 8192
140422024-02-04T14:06:29.054Z07/30 Write block 164 len 28672 data: 9 11 11 11 10 10 11
140432024-02-04T14:06:29.054Z08/30 Read block 472 len 4096
140442024-02-04T14:06:29.054Z09/30 Write block 56 len 4096 data: 10
140452024-02-04T14:06:29.054Z10/30 Read block 500 len 20480
140462024-02-04T14:06:29.054Z11/30 Flush
140472024-02-04T14:06:29.054Z12/30 Write block 502 len 4096 data: 11
140482024-02-04T14:06:29.054Z13/30 Read block 414 len 40960
140492024-02-04T14:06:29.057Z14/30 Write block 118 len 4096 data: 10
140502024-02-04T14:06:29.057Z15/30 Read block 419 len 36864
140512024-02-04T14:06:29.057Z16/30 Read block 470 len 8192
140522024-02-04T14:06:29.057Z17/30 Write block 370 len 4096 data: 13
140532024-02-04T14:06:29.057Z18/30 Write block 200 len 40960 data: 14 14 13 13 14 11 10 9 8 7
140542024-02-04T14:06:29.057Z19/30 Read block 447 len 8192
140552024-02-04T14:06:29.057Z20/30 Read block 188 len 8192
140562024-02-04T14:06:29.057Z21/30 Flush
140572024-02-04T14:06:29.060Z22/30 Write block 494 len 20480 data: 13 12 14 12 14
140582024-02-04T14:06:29.063Z23/30 Write block 310 len 28672 data: 10 10 8 9 9 6 6
140592024-02-04T14:06:29.063Z24/30 Read block 527 len 40960
140602024-02-04T14:06:29.066Z25/30 Read block 170 len 20480
140612024-02-04T14:06:29.066Z26/30 Write block 341 len 12288 data: 9 8 9
140622024-02-04T14:06:29.068Z27/30 Write block 339 len 16384 data: 8 10 10 9
140632024-02-04T14:06:29.069Z28/30 Read block 591 len 24576
140642024-02-04T14:06:29.071Z29/30 Write block 302 len 28672 data: 8 7 8 10 9 9 8
140652024-02-04T14:06:29.071Z30/30 Read block 374 len 8192
140662024-02-04T14:06:29.075Z----------------------------------------------------------------
140672024-02-04T14:06:29.075Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
140682024-02-04T14:06:29.075ZStates: Active Active Active
140692024-02-04T14:06:29.075ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140702024-02-04T14:06:29.075Z 1 Acked 1000 Write 10 Done Sent Done false
140712024-02-04T14:06:29.075Z 2 Acked 1001 Read 9 Done Sent Done false
140722024-02-04T14:06:29.075Z 3 Acked 1002 Read 6 Done Sent Done false
140732024-02-04T14:06:29.075Z 4 Acked 1003 Write 6 Done Sent Done false
140742024-02-04T14:06:29.075Z 5 Acked 1004 Write 3 Done Sent Done false
140752024-02-04T14:06:29.075Z 6 Acked 1005 Read 2 Done Sent Done false
140762024-02-04T14:06:29.075Z 7 Acked 1006 Write 7 Done Sent Done false
140772024-02-04T14:06:29.075Z 8 Acked 1007 Read 1 Done Sent Done false
140782024-02-04T14:06:29.075Z 9 Acked 1008 Write 1 Done Sent Done false
140792024-02-04T14:06:29.075Z 10 Acked 1009 Read 5 Done Sent Done false
140802024-02-04T14:06:29.075Z 11 Acked 1010 Flush 0 Done Sent Done false
140812024-02-04T14:06:29.075Z 12 Acked 1011 Write 1 Done Sent Done false
140822024-02-04T14:06:29.075Z 13 Acked 1012 Read 10 Done Sent Done false
140832024-02-04T14:06:29.075Z 14 Acked 1013 Write 1 Done Sent Done false
140842024-02-04T14:06:29.076Z 15 Acked 1014 Read 9 Done Sent Done false
140852024-02-04T14:06:29.076Z 16 Acked 1015 Read 2 Done Sent Done false
140862024-02-04T14:06:29.076Z 17 Acked 1016 Write 1 Done Sent Done false
140872024-02-04T14:06:29.076Z 18 Acked 1017 Write 10 Done Sent Done false
140882024-02-04T14:06:29.076Z 19 Acked 1018 Read 2 Done Sent Done false
140892024-02-04T14:06:29.076Z 20 Acked 1019 Read 2 Done Sent Done false
140902024-02-04T14:06:29.076Z 21 Acked 1020 Flush 0 Done Sent Done false
140912024-02-04T14:06:29.076Z 22 Acked 1021 Write 5 Done Sent Done false
140922024-02-04T14:06:29.076Z 23 Acked 1022 Write 7 Done Sent Done false
140932024-02-04T14:06:29.076Z 24 Acked 1023 Read 10 Done Sent Done false
140942024-02-04T14:06:29.076Z 25 Acked 1024 Read 5 Done Sent Done false
140952024-02-04T14:06:29.076Z 26 Acked 1025 Write 3 Done Sent Done false
140962024-02-04T14:06:29.076Z 27 Acked 1026 Write 4 Done Sent Done false
140972024-02-04T14:06:29.076Z 28 Acked 1027 Read 6 Done Sent Done false
140982024-02-04T14:06:29.076Z 29 Acked 1028 Write 7 Done Sent Done false
140992024-02-04T14:06:29.076Z 30 Acked 1029 Read 2 Done Sent Done false
141002024-02-04T14:06:29.076Z STATES DS:0 DS:1 DS:2 TOTAL
141012024-02-04T14:06:29.076Z New 0 0 0 0
141022024-02-04T14:06:29.076Z Sent 0 30 0 30
141032024-02-04T14:06:29.076Z Done 30 0 30 60
141042024-02-04T14:06:29.076Z Skipped 0 0 0 0
141052024-02-04T14:06:29.076Z Error 0 0 0 0
141062024-02-04T14:06:29.076ZLast Flush: 1020 0 1020
141072024-02-04T14:06:29.077ZDownstairs last five completed:
141082024-02-04T14:06:29.077ZUpstairs last five completed: 30 29 28 27 26
141092024-02-04T14:06:29.077Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:28.105696306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1649}
141102024-02-04T14:06:29.077ZWrote out file "/tmp/test_repair_verify.data"
141112024-02-04T14:06:29.080Z
141122024-02-04T14:06:29.087ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141132024-02-04T14:06:29.113ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141142024-02-04T14:06:29.113Z 0 000-019 130 123 130 214 204 214 F F F <---
141152024-02-04T14:06:29.113Z 1 020-039 128 128 128 209 209 209 F F F
141162024-02-04T14:06:29.113Z 2 040-059 130 127 130 214 208 214 F F F <---
141172024-02-04T14:06:29.113Z 3 060-079 128 128 128 212 212 212 F F F
141182024-02-04T14:06:29.113Z 4 080-099 128 128 128 212 212 212 F F F
141192024-02-04T14:06:29.113Z 5 100-119 130 127 130 215 208 215 F F F <---
141202024-02-04T14:06:29.113Z 6 120-139 129 129 129 213 213 213 F F F
141212024-02-04T14:06:29.113Z 7 140-159 129 129 129 213 213 213 F F F
141222024-02-04T14:06:29.113Z 8 160-179 130 122 130 214 203 214 F F F <---
141232024-02-04T14:06:29.118Z 9 180-199 128 128 128 210 210 210 F F F
141242024-02-04T14:06:29.118Z 10 200-219 130 128 130 215 212 215 F F F <---
141252024-02-04T14:06:29.119Z 11 220-239 128 128 128 212 212 212 F F F
141262024-02-04T14:06:29.119Z 12 240-259 127 127 127 208 208 208 F F F
141272024-02-04T14:06:29.119Z 13 260-279 124 124 124 205 205 205 F F F
141282024-02-04T14:06:29.119Z 14 280-299 127 127 127 208 208 208 F F F
141292024-02-04T14:06:29.119Z 15 300-319 120 120 120 200 200 200 T F T <---
141302024-02-04T14:06:29.119Z 16 320-339 127 127 127 208 208 208 T F T <---
141312024-02-04T14:06:29.119Z 17 340-359 130 128 130 214 211 214 T F T <---
141322024-02-04T14:06:29.120Z 18 360-379 130 116 130 215 188 215 F F F <---
141332024-02-04T14:06:29.120Z 19 380-399 128 128 128 212 212 212 F F F
141342024-02-04T14:06:29.120Z 20 400-419 130 128 130 214 210 214 F F F <---
141352024-02-04T14:06:29.120Z 21 420-439 127 127 127 208 208 208 F F F
141362024-02-04T14:06:29.120Z 22 440-459 125 125 125 206 206 206 F F F
141372024-02-04T14:06:29.120Z 23 460-479 126 126 126 207 207 207 F F F
141382024-02-04T14:06:29.120Z 24 480-499 128 128 128 212 212 212 T F T <---
141392024-02-04T14:06:29.120Z 25 500-519 130 128 130 215 212 215 F F F <---
141402024-02-04T14:06:29.121Z 26 520-539 125 125 125 206 206 206 F F F
141412024-02-04T14:06:29.121Z 27 540-559 125 125 125 206 206 206 F F F
141422024-02-04T14:06:29.121Z 28 560-579 127 127 127 208 208 208 F F F
141432024-02-04T14:06:29.121Z 29 580-599 124 124 124 205 205 205 F F F
141442024-02-04T14:06:29.121ZMax gen: 130, Max flush: 215
141452024-02-04T14:06:29.121ZError: Difference in extent metadata found!
141462024-02-04T14:06:29.132ZA Difference in extent metadata is expected here
141472024-02-04T14:06:29.132ZOn loop 65
141482024-02-04T14:06:29.133Z
141492024-02-04T14:06:29.133ZVerifying data now
141502024-02-04T14:06:34.191ZNo change after verify
141512024-02-04T14:06:34.191ZLoop: 65 Downstairs dump after verify (and repair):
141522024-02-04T14:06:34.213ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141532024-02-04T14:06:34.213Z 0 000-019 130 130 130 214 214 214 F F F
141542024-02-04T14:06:34.214Z 1 020-039 128 128 128 209 209 209 F F F
141552024-02-04T14:06:34.217Z 2 040-059 130 130 130 214 214 214 F F F
141562024-02-04T14:06:34.217Z 3 060-079 128 128 128 212 212 212 F F F
141572024-02-04T14:06:34.217Z 4 080-099 128 128 128 212 212 212 F F F
141582024-02-04T14:06:34.217Z 5 100-119 130 130 130 215 215 215 F F F
141592024-02-04T14:06:34.217Z 6 120-139 129 129 129 213 213 213 F F F
141602024-02-04T14:06:34.217Z 7 140-159 129 129 129 213 213 213 F F F
141612024-02-04T14:06:34.217Z 8 160-179 130 130 130 214 214 214 F F F
141622024-02-04T14:06:34.217Z 9 180-199 128 128 128 210 210 210 F F F
141632024-02-04T14:06:34.217Z 10 200-219 130 130 130 215 215 215 F F F
141642024-02-04T14:06:34.217Z 11 220-239 128 128 128 212 212 212 F F F
141652024-02-04T14:06:34.217Z 12 240-259 127 127 127 208 208 208 F F F
141662024-02-04T14:06:34.217Z 13 260-279 124 124 124 205 205 205 F F F
141672024-02-04T14:06:34.217Z 14 280-299 127 127 127 208 208 208 F F F
141682024-02-04T14:06:34.217Z 15 300-319 131 131 131 216 216 216 F F F
141692024-02-04T14:06:34.217Z 16 320-339 131 131 131 216 216 216 F F F
141702024-02-04T14:06:34.217Z 17 340-359 131 131 131 216 216 216 F F F
141712024-02-04T14:06:34.217Z 18 360-379 130 130 130 215 215 215 F F F
141722024-02-04T14:06:34.217Z 19 380-399 128 128 128 212 212 212 F F F
141732024-02-04T14:06:34.218Z 20 400-419 130 130 130 214 214 214 F F F
141742024-02-04T14:06:34.218Z 21 420-439 127 127 127 208 208 208 F F F
141752024-02-04T14:06:34.218Z 22 440-459 125 125 125 206 206 206 F F F
141762024-02-04T14:06:34.218Z 23 460-479 126 126 126 207 207 207 F F F
141772024-02-04T14:06:34.218Z 24 480-499 131 131 131 216 216 216 F F F
141782024-02-04T14:06:34.218Z 25 500-519 130 130 130 215 215 215 F F F
141792024-02-04T14:06:34.218Z 26 520-539 125 125 125 206 206 206 F F F
141802024-02-04T14:06:34.218Z 27 540-559 125 125 125 206 206 206 F F F
141812024-02-04T14:06:34.218Z 28 560-579 127 127 127 208 208 208 F F F
141822024-02-04T14:06:34.218Z 29 580-599 124 124 124 205 205 205 F F F
141832024-02-04T14:06:34.218ZMax gen: 131, Max flush: 216
141842024-02-04T14:06:34.223Z
141852024-02-04T14:06:34.223ZBegin loop 66
141862024-02-04T14:06:34.223ZDownstairs to restart: 2
141872024-02-04T14:06:34.253Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.285246804Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657}
141882024-02-04T14:06:34.253Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.285722218Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657}
141892024-02-04T14:06:34.253Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.285751135Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657}
141902024-02-04T14:06:34.253Z{"msg":"Crucible c378bd39-5f61-460c-9a8a-4fc2eb533f37 has session id: 34ad898c-aa29-478e-be79-ce4e6ea61840","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.28577977Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
141912024-02-04T14:06:34.257Z{"msg":"Upstairs opts: Upstairs UUID: c378bd39-5f61-460c-9a8a-4fc2eb533f37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.285887179Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"Crucible runtime is spawned
141922024-02-04T14:06:34.257Z}
141932024-02-04T14:06:34.257Z{"msg":"Crucible stats registered with UUID: c378bd39-5f61-460c-9a8a-4fc2eb533f37","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.285994306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
141942024-02-04T14:06:34.257Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286048088Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657}
141952024-02-04T14:06:34.257Z{"msg":"c378bd39-5f61-460c-9a8a-4fc2eb533f37 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286181525Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
141962024-02-04T14:06:34.257Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286240009Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
141972024-02-04T14:06:34.257Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286283887Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
141982024-02-04T14:06:34.257Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.28635821Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
141992024-02-04T14:06:34.257Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286401353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142002024-02-04T14:06:34.257Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.28649567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142012024-02-04T14:06:34.257Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286542184Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142022024-02-04T14:06:34.257Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.28663889Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142032024-02-04T14:06:34.257Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286684255Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142042024-02-04T14:06:34.257Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286730687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142052024-02-04T14:06:34.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286773088Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142062024-02-04T14:06:34.258Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.286843412Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142072024-02-04T14:06:34.258Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.287015362Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142082024-02-04T14:06:34.258Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.288179878Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142092024-02-04T14:06:34.258Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.288252946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142102024-02-04T14:06:34.258Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.288279755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142112024-02-04T14:06:34.258Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.288302588Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142122024-02-04T14:06:34.258Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.289213391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142132024-02-04T14:06:34.258Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.289448416Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142142024-02-04T14:06:34.262Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.29010778Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142152024-02-04T14:06:34.262Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.290197496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142162024-02-04T14:06:34.262Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.290237872Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142172024-02-04T14:06:34.262Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.290283535Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142182024-02-04T14:06:34.262Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.290308348Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142192024-02-04T14:06:34.262Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.290350341Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142202024-02-04T14:06:34.262Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.290373122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142212024-02-04T14:06:34.263Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.290447934Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142222024-02-04T14:06:34.263Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.291262845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142232024-02-04T14:06:34.263Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.291973324Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142242024-02-04T14:06:34.263Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292067622Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142252024-02-04T14:06:34.263Z{"msg":"[0]R flush_numbers[0..12]: [214, 209, 214, 212, 212, 215, 213, 213, 214, 210, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292101851Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142262024-02-04T14:06:34.263Z{"msg":"[0]R generation[0..12]: [130, 128, 130, 128, 128, 130, 129, 129, 130, 128, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292148231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142272024-02-04T14:06:34.263Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292174604Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142282024-02-04T14:06:34.263Z{"msg":"[1]R flush_numbers[0..12]: [214, 209, 214, 212, 212, 215, 213, 213, 214, 210, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292217366Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142292024-02-04T14:06:34.263Z{"msg":"[1]R generation[0..12]: [130, 128, 130, 128, 128, 130, 129, 129, 130, 128, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292242276Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142302024-02-04T14:06:34.263Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"Wait for a query_work_queue command to finish before sending IO
142312024-02-04T14:06:34.263Zname":"crucible","level":30,"time":"2024-02-04T14:06:33.29228974Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142322024-02-04T14:06:34.263Z{"msg":"[2]R flush_numbers[0..12]: [214, 209, 214, 212, 212, 215, 213, 213, 214, 210, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292315922Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142332024-02-04T14:06:34.263Z{"msg":"[2]R generation[0..12]: [130, 128, 130, 128, 128, 130, 129, 129, 130, 128, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292359816Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142342024-02-04T14:06:34.263Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292385134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142352024-02-04T14:06:34.263Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292432471Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142362024-02-04T14:06:34.263Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292456154Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142372024-02-04T14:06:34.263Z{"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292511418Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142382024-02-04T14:06:34.263Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292534604Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142392024-02-04T14:06:34.263Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292557651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142402024-02-04T14:06:34.263Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292579453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142412024-02-04T14:06:34.263Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292601565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142422024-02-04T14:06:34.263Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.29262328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142432024-02-04T14:06:34.263Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292645316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142442024-02-04T14:06:34.264Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292666237Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142452024-02-04T14:06:34.264Z{"msg":"c378bd39-5f61-460c-9a8a-4fc2eb533f37 is now active with session: 34ad898c-aa29-478e-be79-ce4e6ea61840","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292688095Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142462024-02-04T14:06:34.264Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292726923Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657,"session_id":"34ad898c-aa29-478e-be79-ce4e6ea61840"}
142472024-02-04T14:06:34.264Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:33.292748968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657}
142482024-02-04T14:06:34.264ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142492024-02-04T14:06:34.264ZLoading write count information from file "/tmp/test_repair_verify.data"
142502024-02-04T14:06:34.264ZRun Repair workload
142512024-02-04T14:06:34.264Z01/30 Flush
142522024-02-04T14:06:34.270Z02/30 Write block 78 len 40960 data: 12 11 13 14 15 13 13 13 12 8
142532024-02-04T14:06:34.270Z03/30 Read block 525 len 28672
142542024-02-04T14:06:34.273Z04/30 Write block 48 len 28672 data: 10 7 6 7 7 7 7
142552024-02-04T14:06:34.273Z05/30 Flush
142562024-02-04T14:06:34.273Z06/30 Write block 94 len 12288 data: 5 8 7
142572024-02-04T14:06:34.273Z07/30 Read block 109 len 12288
142582024-02-04T14:06:34.276Z08/30 Read block 435 len 40960
142592024-02-04T14:06:34.280Z09/30 Write block 71 len 12288 data: 8 8 8
142602024-02-04T14:06:34.280Z10/30 Write block 242 len 20480 data: 10 9 10 9 11
142612024-02-04T14:06:34.280Z11/30 Read block 468 len 20480
142622024-02-04T14:06:34.283Z12/30 Read block 436 len 16384
142632024-02-04T14:06:34.283Z13/30 Read block 61 len 24576
142642024-02-04T14:06:34.283Z14/30 Read block 377 len 32768
142652024-02-04T14:06:34.286Z15/30 Flush
142662024-02-04T14:06:34.290Z16/30 Write block 69 len 8192 data: 9 7
142672024-02-04T14:06:34.290Z17/30 Read block 346 len 4096
142682024-02-04T14:06:34.290Z18/30 Read block 49 len 24576
142692024-02-04T14:06:34.293Z19/30 Read block 387 len 36864
142702024-02-04T14:06:34.293Z20/30 Flush
142712024-02-04T14:06:34.296Z21/30 Write block 563 len 16384 data: 12 14 15 14
142722024-02-04T14:06:34.296Z22/30 Write block 163 len 4096 data: 10
142732024-02-04T14:06:34.300Z23/30 Read block 400 len 32768
142742024-02-04T14:06:34.300Z24/30 Flush
142752024-02-04T14:06:34.303Z25/30 Write block 450 len 24576 data: 11 11 12 11 13 14
142762024-02-04T14:06:34.304Z26/30 Write block 304 len 28672 data: 9 11 10 10 9 9 11
142772024-02-04T14:06:34.306Z27/30 Write block 524 len 16384 data: 10 9 12 11
142782024-02-04T14:06:34.306Z28/30 Write block 452 len 8192 data: 13 12
142792024-02-04T14:06:34.306Z29/30 Write block 177 len 36864 data: 14 13 15 14 14 14 16 14 15
142802024-02-04T14:06:34.310Z30/30 Read block 66 len 28672
142812024-02-04T14:06:34.310Z----------------------------------------------------------------
142822024-02-04T14:06:34.310Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:29
142832024-02-04T14:06:34.310ZStates: Active Active Active
142842024-02-04T14:06:34.310ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142852024-02-04T14:06:34.310Z 2 Acked 1001 Write 10 Done Done Done false
142862024-02-04T14:06:34.311Z 3 Acked 1002 Read 7 Done Done Done false
142872024-02-04T14:06:34.311Z 4 Acked 1003 Write 7 Done Done Sent false
142882024-02-04T14:06:34.311Z 5 Acked 1004 Flush 0 Done Done Sent false
142892024-02-04T14:06:34.311Z 6 Acked 1005 Write 3 Done Done Sent false
142902024-02-04T14:06:34.311Z 7 Acked 1006 Read 3 Done Done Sent false
142912024-02-04T14:06:34.311Z 8 Acked 1007 Read 10 Done Done Sent false
142922024-02-04T14:06:34.311Z 9 Acked 1008 Write 3 Done Done Sent false
142932024-02-04T14:06:34.311Z 10 Acked 1009 Write 5 Done Done Sent false
142942024-02-04T14:06:34.311Z 11 Acked 1010 Read 5 Done Done Sent false
142952024-02-04T14:06:34.311Z 12 Acked 1011 Read 4 Done Done Sent false
142962024-02-04T14:06:34.311Z 13 Acked 1012 Read 6 Done Done Sent false
142972024-02-04T14:06:34.311Z 14 Acked 1013 Read 8 Done Done Sent false
142982024-02-04T14:06:34.311Z 15 Acked 1014 Flush 0 Done Done Sent false
142992024-02-04T14:06:34.311Z 16 Acked 1015 Write 2 Done Done Sent false
143002024-02-04T14:06:34.311Z 17 Acked 1016 Read 1 Done Done Sent false
143012024-02-04T14:06:34.311Z 18 Acked 1017 Read 6 Done Done Sent false
143022024-02-04T14:06:34.311Z 19 Acked 1018 Read 9 Done Done Sent false
143032024-02-04T14:06:34.311Z 20 Acked 1019 Flush 0 Done Done Sent false
143042024-02-04T14:06:34.311Z 21 Acked 1020 Write 4 Done Done Sent false
143052024-02-04T14:06:34.311Z 22 Acked 1021 Write 1 Done Done Sent false
143062024-02-04T14:06:34.311Z 23 Acked 1022 Read 8 Done Done Sent false
143072024-02-04T14:06:34.311Z 24 Acked 1023 Flush 0 Done Done Sent false
143082024-02-04T14:06:34.311Z 25 Acked 1024 Write 6 Done Done Sent false
143092024-02-04T14:06:34.312Z 26 Acked 1025 Write 7 Done Done Sent false
143102024-02-04T14:06:34.312Z 27 Acked 1026 Write 4 Done Done Sent false
143112024-02-04T14:06:34.315Z 28 Acked 1027 Write 2 Done Done Sent false
143122024-02-04T14:06:34.315Z 29 Acked 1028 Write 9 Done Done Sent false
143132024-02-04T14:06:34.315Z 30 Acked 1029 Read 7 Done Sent Sent false
143142024-02-04T14:06:34.315Z STATES DS:0 DS:1 DS:2 TOTAL
143152024-02-04T14:06:34.315Z New 0 0 0 0
143162024-02-04T14:06:34.315Z Sent 0 1 27 28
143172024-02-04T14:06:34.315Z Done 29 28 2 59
143182024-02-04T14:06:34.315Z Skipped 0 0 0 0
143192024-02-04T14:06:34.315Z Error 0 0 0 0
143202024-02-04T14:06:34.315ZLast Flush: 1023 1023 1000
143212024-02-04T14:06:34.315ZDownstairs last five completed: 1000
143222024-02-04T14:06:34.315ZUpstairs last five completed: 30 29 28 27 26
143232024-02-04T14:06:34.315Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:33.343412267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1657}
143242024-02-04T14:06:34.316ZWrote out file "/tmp/test_repair_verify.data"
143252024-02-04T14:06:34.320Z
143262024-02-04T14:06:34.324ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143272024-02-04T14:06:34.367ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143282024-02-04T14:06:34.367Z 0 000-019 130 130 130 214 214 214 F F F
143292024-02-04T14:06:34.367Z 1 020-039 128 128 128 209 209 209 F F F
143302024-02-04T14:06:34.367Z 2 040-059 132 132 130 218 218 214 F F F <---
143312024-02-04T14:06:34.367Z 3 060-079 132 132 128 220 220 212 F F T <---
143322024-02-04T14:06:34.367Z 4 080-099 132 132 128 219 219 212 F F T <---
143332024-02-04T14:06:34.367Z 5 100-119 130 130 130 215 215 215 F F F
143342024-02-04T14:06:34.382Z 6 120-139 129 129 129 213 213 213 F F F
143352024-02-04T14:06:34.382Z 7 140-159 129 129 129 213 213 213 F F F
143362024-02-04T14:06:34.383Z 8 160-179 132 132 130 221 221 214 T T F <---
143372024-02-04T14:06:34.383Z 9 180-199 128 128 128 210 210 210 T T F <---
143382024-02-04T14:06:34.383Z 10 200-219 130 130 130 215 215 215 F F F
143392024-02-04T14:06:34.383Z 11 220-239 128 128 128 212 212 212 F F F
143402024-02-04T14:06:34.383Z 12 240-259 132 132 127 219 219 208 F F F <---
143412024-02-04T14:06:34.383Z 13 260-279 124 124 124 205 205 205 F F F
143422024-02-04T14:06:34.383Z 14 280-299 127 127 127 208 208 208 F F F
143432024-02-04T14:06:34.383Z 15 300-319 131 131 131 216 216 216 T T F <---
143442024-02-04T14:06:34.383Z 16 320-339 131 131 131 216 216 216 F F F
143452024-02-04T14:06:34.383Z 17 340-359 131 131 131 216 216 216 F F F
143462024-02-04T14:06:34.383Z 18 360-379 130 130 130 215 215 215 F F F
143472024-02-04T14:06:34.384Z 19 380-399 128 128 128 212 212 212 F F F
143482024-02-04T14:06:34.384Z 20 400-419 130 130 130 214 214 214 F F F
143492024-02-04T14:06:34.384Z 21 420-439 127 127 127 208 208 208 F F F
143502024-02-04T14:06:34.384Z 22 440-459 125 125 125 206 206 206 T T F <---
143512024-02-04T14:06:34.384Z 23 460-479 126 126 126 207 207 207 F F F
143522024-02-04T14:06:34.384Z 24 480-499 131 131 131 216 216 216 F F F
143532024-02-04T14:06:34.384Z 25 500-519 130 130 130 215 215 215 F F F
143542024-02-04T14:06:34.384Z 26 520-539 125 125 125 206 206 206 T T F <---
143552024-02-04T14:06:34.384Z 27 540-559 125 125 125 206 206 206 F F F
143562024-02-04T14:06:34.384Z 28 560-579 132 132 127 221 221 208 F F F <---
143572024-02-04T14:06:34.384Z 29 580-599 124 124 124 205 205 205 F F F
143582024-02-04T14:06:34.384ZMax gen: 132, Max flush: 221
143592024-02-04T14:06:34.384ZError: Difference in extent metadata found!
143602024-02-04T14:06:34.384ZA Difference in extent metadata is expected here
143612024-02-04T14:06:34.384ZOn loop 66
143622024-02-04T14:06:34.384Z
143632024-02-04T14:06:34.384ZVerifying data now
143642024-02-04T14:06:40.286ZNo change after verify
143652024-02-04T14:06:40.286ZLoop: 66 Downstairs dump after verify (and repair):
143662024-02-04T14:06:40.309ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143672024-02-04T14:06:40.309Z 0 000-019 130 130 130 214 214 214 F F F
143682024-02-04T14:06:40.309Z 1 020-039 128 128 128 209 209 209 F F F
143692024-02-04T14:06:40.309Z 2 040-059 132 132 132 218 218 218 F F F
143702024-02-04T14:06:40.309Z 3 060-079 132 132 132 220 220 220 F F F
143712024-02-04T14:06:40.309Z 4 080-099 132 132 132 219 219 219 F F F
143722024-02-04T14:06:40.309Z 5 100-119 130 130 130 215 215 215 F F F
143732024-02-04T14:06:40.310Z 6 120-139 129 129 129 213 213 213 F F F
143742024-02-04T14:06:40.310Z 7 140-159 129 129 129 213 213 213 F F F
143752024-02-04T14:06:40.310Z 8 160-179 133 133 133 222 222 222 F F F
143762024-02-04T14:06:40.310Z 9 180-199 133 133 133 222 222 222 F F F
143772024-02-04T14:06:40.310Z 10 200-219 130 130 130 215 215 215 F F F
143782024-02-04T14:06:40.319Z 11 220-239 128 128 128 212 212 212 F F F
143792024-02-04T14:06:40.319Z 12 240-259 132 132 132 219 219 219 F F F
143802024-02-04T14:06:40.319Z 13 260-279 124 124 124 205 205 205 F F F
143812024-02-04T14:06:40.319Z 14 280-299 127 127 127 208 208 208 F F F
143822024-02-04T14:06:40.319Z 15 300-319 133 133 133 222 222 222 F F F
143832024-02-04T14:06:40.320Z 16 320-339 131 131 131 216 216 216 F F F
143842024-02-04T14:06:40.320Z 17 340-359 131 131 131 216 216 216 F F F
143852024-02-04T14:06:40.320Z 18 360-379 130 130 130 215 215 215 F F F
143862024-02-04T14:06:40.320Z 19 380-399 128 128 128 212 212 212 F F F
143872024-02-04T14:06:40.320Z 20 400-419 130 130 130 214 214 214 F F F
143882024-02-04T14:06:40.320Z 21 420-439 127 127 127 208 208 208 F F F
143892024-02-04T14:06:40.320Z 22 440-459 133 133 133 222 222 222 F F F
143902024-02-04T14:06:40.320Z 23 460-479 126 126 126 207 207 207 F F F
143912024-02-04T14:06:40.320Z 24 480-499 131 131 131 216 216 216 F F F
143922024-02-04T14:06:40.320Z 25 500-519 130 130 130 215 215 215 F F F
143932024-02-04T14:06:40.320Z 26 520-539 133 133 133 222 222 222 F F F
143942024-02-04T14:06:40.321Z 27 540-559 125 125 125 206 206 206 F F F
143952024-02-04T14:06:40.321Z 28 560-579 132 132 132 221 221 221 F F F
143962024-02-04T14:06:40.321Z 29 580-599 124 124 124 205 205 205 F F F
143972024-02-04T14:06:40.321ZMax gen: 133, Max flush: 222
143982024-02-04T14:06:40.321Z
143992024-02-04T14:06:40.321ZBegin loop 67
144002024-02-04T14:06:40.321ZDownstairs to restart: 0
144012024-02-04T14:06:40.347Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.376863727Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665}
144022024-02-04T14:06:40.347Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.377369784Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665}
144032024-02-04T14:06:40.347Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.377402293Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665}
144042024-02-04T14:06:40.348Z{"msg":"Crucible 996abca3-9ecb-48ea-a0cd-b5a7399cc9ce has session id: 2722bccb-37d2-4dbf-a8f4-cf537264b3f5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.37746951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144052024-02-04T14:06:40.361Z{"msg":"Upstairs opts: Upstairs UUID: 996abca3-9ecb-48ea-a0cd-b5a7399cc9ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.377529707Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144062024-02-04T14:06:40.361Z{"msg":"Crucible stats registered with UUID: 996abca3-9ecb-48ea-a0cd-b5a7399cc9ce","v":0,"name":"crucible","level":30Crucible runtime is spawned
144072024-02-04T14:06:40.361Z,"time":"2024-02-04T14:06:39.377632686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144082024-02-04T14:06:40.361Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.377821952Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665}
144092024-02-04T14:06:40.361Z{"msg":"996abca3-9ecb-48ea-a0cd-b5a7399cc9ce active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.37793911Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144102024-02-04T14:06:40.361Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378011364Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144112024-02-04T14:06:40.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378056797Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144122024-02-04T14:06:40.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378140287Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144132024-02-04T14:06:40.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.37818587Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144142024-02-04T14:06:40.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378287222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144152024-02-04T14:06:40.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378333739Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144162024-02-04T14:06:40.362Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378404145Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144172024-02-04T14:06:40.362Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378449198Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144182024-02-04T14:06:40.362Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378599959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144192024-02-04T14:06:40.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.37869175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144202024-02-04T14:06:40.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.378731269Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144212024-02-04T14:06:40.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.37877191Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144222024-02-04T14:06:40.362Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.379805013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144232024-02-04T14:06:40.363Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.379861812Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144242024-02-04T14:06:40.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.379886623Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144252024-02-04T14:06:40.363Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.379986183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144262024-02-04T14:06:40.363Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.380821759Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144272024-02-04T14:06:40.363Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.38111701Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144282024-02-04T14:06:40.363Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.381794033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144292024-02-04T14:06:40.363Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.38187569Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144302024-02-04T14:06:40.364Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.381946092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144312024-02-04T14:06:40.364Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.381986089Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144322024-02-04T14:06:40.364Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.382057423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144332024-02-04T14:06:40.364Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.382098683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144342024-02-04T14:06:40.364Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.382334795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144352024-02-04T14:06:40.364Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.382394168Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144362024-02-04T14:06:40.365Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.383120825Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144372024-02-04T14:06:40.365Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.383796303Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144382024-02-04T14:06:40.365Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.383847264Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144392024-02-04T14:06:40.365Z{"msg":"[0]R flush_numbers[0..12]: [214, 209, 218, 220, 219, 215, 213, 213, 222, 222, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.383878535Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144402024-02-04T14:06:40.365Z{"msg":"[0]R generation[0..12]: [130, 128, 132, 132, 132, 130, 129, 129, 133, 133, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.38392025Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144412024-02-04T14:06:40.365Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.38396133Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144422024-02-04T14:06:40.366Z{"msg":"[1]R flush_numbers[0..12]: [214, 209, 218, 220, 219, 215, 213, 213, 222, 222, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.3839978Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144432024-02-04T14:06:40.366Z{"msg":"[1]R generation[0..12]: [130, 128, 132, 132, 132, 130, 129, 129, 133, 133, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384055591Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144442024-02-04T14:06:40.366Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.38408326Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144452024-02-04T14:06:40.366Z{"msg":"[2]R flush_numbers[0..12]: [214, 209, 218, 220, 219, 215, 213, 213, 222, 222, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384140312Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144462024-02-04T14:06:40.366Z{"msg":"[2]R generation[0..12]: [130, 128, 132, 132, 132, 130, 129, 129, 133, 133, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384165156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144472024-02-04T14:06:40.366Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
144482024-02-04T14:06:40.366Z,"time":"2024-02-04T14:06:39.384190009Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144492024-02-04T14:06:40.367Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384212354Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144502024-02-04T14:06:40.367Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384235293Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144512024-02-04T14:06:40.367Z{"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384258479Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144522024-02-04T14:06:40.367Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384329706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144532024-02-04T14:06:40.367Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384371799Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144542024-02-04T14:06:40.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384393561Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144552024-02-04T14:06:40.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.38441609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144562024-02-04T14:06:40.367Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384438087Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144572024-02-04T14:06:40.368Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384460663Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144582024-02-04T14:06:40.368Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384500554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144592024-02-04T14:06:40.368Z{"msg":"996abca3-9ecb-48ea-a0cd-b5a7399cc9ce is now active with session: 2722bccb-37d2-4dbf-a8f4-cf537264b3f5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384523474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144602024-02-04T14:06:40.368Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.384582152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665,"session_id":"2722bccb-37d2-4dbf-a8f4-cf537264b3f5"}
144612024-02-04T14:06:40.368Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:39.38460446Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665}
144622024-02-04T14:06:40.368ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144632024-02-04T14:06:40.368ZLoading write count information from file "/tmp/test_repair_verify.data"
144642024-02-04T14:06:40.369ZRun Repair workload
144652024-02-04T14:06:40.369Z01/30 Write block 318 len 24576 data: 9 9 8 7 7 6
144662024-02-04T14:06:40.379Z02/30 Read block 216 len 20480
144672024-02-04T14:06:40.379Z03/30 Write block 287 len 28672 data: 14 14 14 13 7 9 6
144682024-02-04T14:06:40.379Z04/30 Read block 2 len 32768
144692024-02-04T14:06:40.379Z05/30 Write block 131 len 4096 data: 5
144702024-02-04T14:06:40.379Z06/30 Flush
144712024-02-04T14:06:40.379Z07/30 Write block 554 len 16384 data: 10 10 12 13
144722024-02-04T14:06:40.379Z08/30 Write block 535 len 4096 data: 6
144732024-02-04T14:06:40.379Z09/30 Read block 121 len 20480
144742024-02-04T14:06:40.379Z10/30 Read block 349 len 16384
144752024-02-04T14:06:40.379Z11/30 Read block 97 len 32768
144762024-02-04T14:06:40.379Z12/30 Read block 541 len 32768
144772024-02-04T14:06:40.397Z13/30 Read block 129 len 24576
144782024-02-04T14:06:40.397Z14/30 Write block 538 len 40960 data: 6 7 8 8 7 10 8 8 6 6
144792024-02-04T14:06:40.397Z15/30 Read block 248 len 32768
144802024-02-04T14:06:40.398Z16/30 Read block 3 len 40960
144812024-02-04T14:06:40.398Z17/30 Write block 22 len 12288 data: 11 12 11
144822024-02-04T14:06:40.398Z18/30 Write block 121 len 36864 data: 12 11 10 9 11 11 10 9 7
144832024-02-04T14:06:40.398Z19/30 Read block 50 len 20480
144842024-02-04T14:06:40.403Z20/30 Write block 39 len 28672 data: 8 7 7 9 8 10 10
144852024-02-04T14:06:40.403Z21/30 Flush
144862024-02-04T14:06:40.403Z22/30 Write block 412 len 8192 data: 12 12
144872024-02-04T14:06:40.404Z23/30 Write block 108 len 32768 data: 13 12 12 13 16 13 15 15
144882024-02-04T14:06:40.404Z24/30 Write block 506 len 4096 data: 15
144892024-02-04T14:06:40.404Z25/30 Write block 261 len 12288 data: 12 13 13
144902024-02-04T14:06:40.404Z26/30 Read block 585 len 36864
144912024-02-04T14:06:40.404Z27/30 Read block 297 len 16384
144922024-02-04T14:06:40.404Z28/30 Write block 300 len 4096 data: 7
144932024-02-04T14:06:40.404Z29/30 Read block 508 len 28672
144942024-02-04T14:06:40.404Z30/30 Read block 499 len 32768
144952024-02-04T14:06:40.408Z----------------------------------------------------------------
144962024-02-04T14:06:40.409Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144972024-02-04T14:06:40.409ZStates: Active Active Active
144982024-02-04T14:06:40.409ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144992024-02-04T14:06:40.409Z 1 Acked 1000 Write 6 Done Done Done false
145002024-02-04T14:06:40.409Z 2 Acked 1001 Read 5 Done Done Done false
145012024-02-04T14:06:40.409Z 3 Acked 1002 Write 7 Done Done Done false
145022024-02-04T14:06:40.409Z 4 Acked 1003 Read 8 Done Done Done false
145032024-02-04T14:06:40.409Z 5 Acked 1004 Write 1 Sent Done Done false
145042024-02-04T14:06:40.409Z 6 Acked 1005 Flush 0 Sent Done Done false
145052024-02-04T14:06:40.409Z 7 Acked 1006 Write 4 Sent Done Done false
145062024-02-04T14:06:40.410Z 8 Acked 1007 Write 1 Sent Done Done false
145072024-02-04T14:06:40.410Z 9 Acked 1008 Read 5 Sent Done Done false
145082024-02-04T14:06:40.410Z 10 Acked 1009 Read 4 Sent Done Done false
145092024-02-04T14:06:40.410Z 11 Acked 1010 Read 8 Sent Done Done false
145102024-02-04T14:06:40.410Z 12 Acked 1011 Read 8 Sent Done Done false
145112024-02-04T14:06:40.410Z 13 Acked 1012 Read 6 Sent Done Done false
145122024-02-04T14:06:40.410Z 14 Acked 1013 Write 10 Sent Done Done false
145132024-02-04T14:06:40.410Z 15 Acked 1014 Read 8 Sent Done Done false
145142024-02-04T14:06:40.410Z 16 Acked 1015 Read 10 Sent Done Done false
145152024-02-04T14:06:40.410Z 17 Acked 1016 Write 3 Sent Done Done false
145162024-02-04T14:06:40.410Z 18 Acked 1017 Write 9 Sent Done Done false
145172024-02-04T14:06:40.411Z 19 Acked 1018 Read 5 Sent Done Done false
145182024-02-04T14:06:40.411Z 20 Acked 1019 Write 7 Sent Done Done false
145192024-02-04T14:06:40.411Z 21 Acked 1020 Flush 0 Sent Done Done false
145202024-02-04T14:06:40.411Z 22 Acked 1021 Write 2 Sent Done Done false
145212024-02-04T14:06:40.411Z 23 Acked 1022 Write 8 Sent Done Done false
145222024-02-04T14:06:40.411Z 24 Acked 1023 Write 1 Sent Done Done false
145232024-02-04T14:06:40.411Z 25 Acked 1024 Write 3 Sent Done Done false
145242024-02-04T14:06:40.411Z 26 Acked 1025 Read 9 Sent Done Done false
145252024-02-04T14:06:40.411Z 27 Acked 1026 Read 4 Sent Done Done false
145262024-02-04T14:06:40.411Z 28 Acked 1027 Write 1 Sent Done Done false
145272024-02-04T14:06:40.411Z 29 Acked 1028 Read 7 Sent Done Done false
145282024-02-04T14:06:40.411Z 30 Acked 1029 Read 8 Sent Done Done false
145292024-02-04T14:06:40.412Z STATES DS:0 DS:1 DS:2 TOTAL
145302024-02-04T14:06:40.412Z New 0 0 0 0
145312024-02-04T14:06:40.412Z Sent 26 0 0 26
145322024-02-04T14:06:40.412Z Done 4 30 30 64
145332024-02-04T14:06:40.412Z Skipped 0 0 0 0
145342024-02-04T14:06:40.412Z Error 0 0 0 0
145352024-02-04T14:06:40.412ZLast Flush: 0 1020 1020
145362024-02-04T14:06:40.412ZDownstairs last five completed:
145372024-02-04T14:06:40.412ZUpstairs last five completed: 30 29 28 27 26
145382024-02-04T14:06:40.412Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:39.435912649Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1665}
145392024-02-04T14:06:40.412ZWrote out file "/tmp/test_repair_verify.data"
145402024-02-04T14:06:40.417Z
145412024-02-04T14:06:40.420ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145422024-02-04T14:06:40.445ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145432024-02-04T14:06:40.445Z 0 000-019 130 130 130 214 214 214 F F F
145442024-02-04T14:06:40.445Z 1 020-039 128 134 134 209 224 224 F F F <---
145452024-02-04T14:06:40.445Z 2 040-059 132 134 134 218 224 224 F F F <---
145462024-02-04T14:06:40.445Z 3 060-079 132 132 132 220 220 220 F F F
145472024-02-04T14:06:40.445Z 4 080-099 132 132 132 219 219 219 F F F
145482024-02-04T14:06:40.445Z 5 100-119 130 130 130 215 215 215 F T T <---
145492024-02-04T14:06:40.445Z 6 120-139 129 134 134 213 224 224 F F F <---
145502024-02-04T14:06:40.446Z 7 140-159 129 129 129 213 213 213 F F F
145512024-02-04T14:06:40.446Z 8 160-179 133 133 133 222 222 222 F F F
145522024-02-04T14:06:40.446Z 9 180-199 133 133 133 222 222 222 F F F
145532024-02-04T14:06:40.446Z 10 200-219 130 130 130 215 215 215 F F F
145542024-02-04T14:06:40.449Z 11 220-239 128 128 128 212 212 212 F F F
145552024-02-04T14:06:40.449Z 12 240-259 132 132 132 219 219 219 F F F
145562024-02-04T14:06:40.449Z 13 260-279 124 124 124 205 205 205 F T T <---
145572024-02-04T14:06:40.449Z 14 280-299 127 134 134 208 223 223 T F F <---
145582024-02-04T14:06:40.449Z 15 300-319 133 134 134 222 223 223 T T T <---
145592024-02-04T14:06:40.449Z 16 320-339 131 134 134 216 223 223 T F F <---
145602024-02-04T14:06:40.449Z 17 340-359 131 131 131 216 216 216 F F F
145612024-02-04T14:06:40.449Z 18 360-379 130 130 130 215 215 215 F F F
145622024-02-04T14:06:40.449Z 19 380-399 128 128 128 212 212 212 F F F
145632024-02-04T14:06:40.449Z 20 400-419 130 130 130 214 214 214 F T T <---
145642024-02-04T14:06:40.449Z 21 420-439 127 127 127 208 208 208 F F F
145652024-02-04T14:06:40.449Z 22 440-459 133 133 133 222 222 222 F F F
145662024-02-04T14:06:40.449Z 23 460-479 126 126 126 207 207 207 F F F
145672024-02-04T14:06:40.449Z 24 480-499 131 131 131 216 216 216 F F F
145682024-02-04T14:06:40.449Z 25 500-519 130 130 130 215 215 215 F T T <---
145692024-02-04T14:06:40.449Z 26 520-539 133 134 134 222 224 224 F F F <---
145702024-02-04T14:06:40.449Z 27 540-559 125 134 134 206 224 224 F F F <---
145712024-02-04T14:06:40.449Z 28 560-579 132 132 132 221 221 221 F F F
145722024-02-04T14:06:40.450Z 29 580-599 124 124 124 205 205 205 F F F
145732024-02-04T14:06:40.450ZMax gen: 134, Max flush: 224
145742024-02-04T14:06:40.450ZError: Difference in extent metadata found!
145752024-02-04T14:06:40.453ZA Difference in extent metadata is expected here
145762024-02-04T14:06:40.453ZOn loop 67
145772024-02-04T14:06:40.453Z
145782024-02-04T14:06:40.456ZVerifying data now
145792024-02-04T14:06:46.531ZNo change after verify
145802024-02-04T14:06:46.531ZLoop: 67 Downstairs dump after verify (and repair):
145812024-02-04T14:06:46.555ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145822024-02-04T14:06:46.555Z 0 000-019 130 130 130 214 214 214 F F F
145832024-02-04T14:06:46.555Z 1 020-039 134 134 134 224 224 224 F F F
145842024-02-04T14:06:46.555Z 2 040-059 134 134 134 224 224 224 F F F
145852024-02-04T14:06:46.555Z 3 060-079 132 132 132 220 220 220 F F F
145862024-02-04T14:06:46.555Z 4 080-099 132 132 132 219 219 219 F F F
145872024-02-04T14:06:46.555Z 5 100-119 135 135 135 225 225 225 F F F
145882024-02-04T14:06:46.555Z 6 120-139 134 134 134 224 224 224 F F F
145892024-02-04T14:06:46.555Z 7 140-159 129 129 129 213 213 213 F F F
145902024-02-04T14:06:46.559Z 8 160-179 133 133 133 222 222 222 F F F
145912024-02-04T14:06:46.559Z 9 180-199 133 133 133 222 222 222 F F F
145922024-02-04T14:06:46.559Z 10 200-219 130 130 130 215 215 215 F F F
145932024-02-04T14:06:46.559Z 11 220-239 128 128 128 212 212 212 F F F
145942024-02-04T14:06:46.559Z 12 240-259 132 132 132 219 219 219 F F F
145952024-02-04T14:06:46.559Z 13 260-279 135 135 135 225 225 225 F F F
145962024-02-04T14:06:46.560Z 14 280-299 134 134 134 223 223 223 F F F
145972024-02-04T14:06:46.560Z 15 300-319 135 135 135 225 225 225 F F F
145982024-02-04T14:06:46.560Z 16 320-339 134 134 134 223 223 223 F F F
145992024-02-04T14:06:46.560Z 17 340-359 131 131 131 216 216 216 F F F
146002024-02-04T14:06:46.560Z 18 360-379 130 130 130 215 215 215 F F F
146012024-02-04T14:06:46.560Z 19 380-399 128 128 128 212 212 212 F F F
146022024-02-04T14:06:46.560Z 20 400-419 135 135 135 225 225 225 F F F
146032024-02-04T14:06:46.560Z 21 420-439 127 127 127 208 208 208 F F F
146042024-02-04T14:06:46.560Z 22 440-459 133 133 133 222 222 222 F F F
146052024-02-04T14:06:46.560Z 23 460-479 126 126 126 207 207 207 F F F
146062024-02-04T14:06:46.560Z 24 480-499 131 131 131 216 216 216 F F F
146072024-02-04T14:06:46.560Z 25 500-519 135 135 135 225 225 225 F F F
146082024-02-04T14:06:46.560Z 26 520-539 134 134 134 224 224 224 F F F
146092024-02-04T14:06:46.561Z 27 540-559 134 134 134 224 224 224 F F F
146102024-02-04T14:06:46.561Z 28 560-579 132 132 132 221 221 221 F F F
146112024-02-04T14:06:46.561Z 29 580-599 124 124 124 205 205 205 F F F
146122024-02-04T14:06:46.561ZMax gen: 135, Max flush: 225
146132024-02-04T14:06:46.564Z
146142024-02-04T14:06:46.564ZBegin loop 68
146152024-02-04T14:06:46.565ZDownstairs to restart: 2
146162024-02-04T14:06:46.595Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.626976803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673}
146172024-02-04T14:06:46.595Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.627441788Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673}
146182024-02-04T14:06:46.595Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.627471506Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673}
146192024-02-04T14:06:46.599Z{"msg":"Crucible fd76e885-4a44-45cc-9ab5-f4dd0625fb78 has session id: 878020ad-d671-4e7a-a947-1a23dbc25266"Crucible runtime is spawned
146202024-02-04T14:06:46.599Z,"v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.62771822Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146212024-02-04T14:06:46.599Z{"msg":"Upstairs opts: Upstairs UUID: fd76e885-4a44-45cc-9ab5-f4dd0625fb78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.62783997Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146222024-02-04T14:06:46.599Z{"msg":"Crucible stats registered with UUID: fd76e885-4a44-45cc-9ab5-f4dd0625fb78","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.627885317Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146232024-02-04T14:06:46.599Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.627943728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673}
146242024-02-04T14:06:46.599Z{"msg":"fd76e885-4a44-45cc-9ab5-f4dd0625fb78 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628019659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146252024-02-04T14:06:46.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628076654Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146262024-02-04T14:06:46.599Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.62812033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146272024-02-04T14:06:46.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628177064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146282024-02-04T14:06:46.599Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628246431Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146292024-02-04T14:06:46.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628289618Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146302024-02-04T14:06:46.600Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628333211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146312024-02-04T14:06:46.600Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628362808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146322024-02-04T14:06:46.600Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628398808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146332024-02-04T14:06:46.600Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628470422Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146342024-02-04T14:06:46.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628691086Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146352024-02-04T14:06:46.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.628811603Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146362024-02-04T14:06:46.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.62886033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146372024-02-04T14:06:46.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.629955013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146382024-02-04T14:06:46.600Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.630026409Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146392024-02-04T14:06:46.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.630115169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146402024-02-04T14:06:46.600Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.630164715Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146412024-02-04T14:06:46.600Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.630899737Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146422024-02-04T14:06:46.605Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.631209559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146432024-02-04T14:06:46.605Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.631622044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146442024-02-04T14:06:46.605Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.631721883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146452024-02-04T14:06:46.605Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.631894747Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146462024-02-04T14:06:46.605Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.631983895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146472024-02-04T14:06:46.605Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.632035219Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146482024-02-04T14:06:46.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.632259514Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146492024-02-04T14:06:46.606Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.63238731Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146502024-02-04T14:06:46.606Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.632430578Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146512024-02-04T14:06:46.606Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.632641149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146522024-02-04T14:06:46.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633366859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146532024-02-04T14:06:46.606Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633427754Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146542024-02-04T14:06:46.606Z{"msg":"[0]R flush_numbers[0..12]: [214, 224, 224, 220, 219, 225, 224, 213, 222, 222, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633454645Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146552024-02-04T14:06:46.606Z{"msg":"[0]R generation[0..12]: [130, 134, 134, 132, 132, 135, 134, 129, 133, 133, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633478175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146562024-02-04T14:06:46.606Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.63352085Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146572024-02-04T14:06:46.606Z{"msg":"[1]R flush_numbers[0..12]: [214, 224, 224, 220, 219, 225, 224, 213, 222, 222, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633546221Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146582024-02-04T14:06:46.606Z{"msg":"[1]R generation[0..12]: [130, 134, 134, 132, 132, 135, 134, 129, 133, 133, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633569486Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146592024-02-04T14:06:46.606Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":Wait for a query_work_queue command to finish before sending IO
146602024-02-04T14:06:46.607Z0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633618049Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146612024-02-04T14:06:46.607Z{"msg":"[2]R flush_numbers[0..12]: [214, 224, 224, 220, 219, 225, 224, 213, 222, 222, 215, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633643261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146622024-02-04T14:06:46.607Z{"msg":"[2]R generation[0..12]: [130, 134, 134, 132, 132, 135, 134, 129, 133, 133, 130, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633666529Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146632024-02-04T14:06:46.607Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633690106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146642024-02-04T14:06:46.607Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633712144Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146652024-02-04T14:06:46.607Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633769461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146662024-02-04T14:06:46.607Z{"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633801163Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146672024-02-04T14:06:46.607Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633845265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146682024-02-04T14:06:46.607Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633879307Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146692024-02-04T14:06:46.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633941283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146702024-02-04T14:06:46.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.633969842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146712024-02-04T14:06:46.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.634026012Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146722024-02-04T14:06:46.607Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.634096206Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146732024-02-04T14:06:46.608Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.634120977Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146742024-02-04T14:06:46.608Z{"msg":"fd76e885-4a44-45cc-9ab5-f4dd0625fb78 is now active with session: 878020ad-d671-4e7a-a947-1a23dbc25266","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.634143183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146752024-02-04T14:06:46.608Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.634171172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673,"session_id":"878020ad-d671-4e7a-a947-1a23dbc25266"}
146762024-02-04T14:06:46.608Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:45.634236208Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673}
146772024-02-04T14:06:46.608ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146782024-02-04T14:06:46.608ZLoading write count information from file "/tmp/test_repair_verify.data"
146792024-02-04T14:06:46.608ZRun Repair workload
146802024-02-04T14:06:46.608Z01/30 Flush
146812024-02-04T14:06:46.608Z02/30 Write block 442 len 32768 data: 9 11 12 12 13 13 12 11
146822024-02-04T14:06:46.624Z03/30 Read block 71 len 32768
146832024-02-04T14:06:46.624Z04/30 Write block 471 len 12288 data: 7 8 8
146842024-02-04T14:06:46.624Z05/30 Read block 340 len 40960
146852024-02-04T14:06:46.624Z06/30 Read block 161 len 40960
146862024-02-04T14:06:46.633Z07/30 Read block 298 len 20480
146872024-02-04T14:06:46.633Z08/30 Flush
146882024-02-04T14:06:46.633Z09/30 Write block 181 len 16384 data: 15 15 17 15
146892024-02-04T14:06:46.633Z10/30 Write block 202 len 24576 data: 14 14 15 12 11 10
146902024-02-04T14:06:46.633Z11/30 Read block 397 len 40960
146912024-02-04T14:06:46.633Z12/30 Write block 204 len 20480 data: 16 13 12 11 9
146922024-02-04T14:06:46.633Z13/30 Read block 11 len 4096
146932024-02-04T14:06:46.633Z14/30 Flush
146942024-02-04T14:06:46.636Z15/30 Write block 186 len 40960 data: 13 13 11 11 9 10 10 10 8 7
146952024-02-04T14:06:46.636Z16/30 Read block 520 len 4096
146962024-02-04T14:06:46.636Z17/30 Write block 456 len 24576 data: 13 13 12 12 10 9
146972024-02-04T14:06:46.636Z18/30 Read block 26 len 12288
146982024-02-04T14:06:46.636Z19/30 Read block 391 len 32768
146992024-02-04T14:06:46.636Z20/30 Flush
147002024-02-04T14:06:46.640Z21/30 Write block 400 len 28672 data: 12 10 12 11 11 14 13
147012024-02-04T14:06:46.640Z22/30 Read block 132 len 8192
147022024-02-04T14:06:46.643Z23/30 Read block 484 len 8192
147032024-02-04T14:06:46.643Z24/30 Read block 559 len 20480
147042024-02-04T14:06:46.646Z25/30 Read block 597 len 8192
147052024-02-04T14:06:46.646Z26/30 Write block 65 len 40960 data: 10 9 12 12 10 8 9 9 9 9
147062024-02-04T14:06:46.649Z27/30 Write block 383 len 40960 data: 8 9 9 8 10 8 9 7 12 13
147072024-02-04T14:06:46.649Z28/30 Write block 98 len 40960 data: 8 9 6 7 7 6 7 7 7 8
147082024-02-04T14:06:46.652Z29/30 Read block 442 len 24576
147092024-02-04T14:06:46.655Z30/30 Write block 397 len 8192 data: 13 11
147102024-02-04T14:06:46.655Z----------------------------------------------------------------
147112024-02-04T14:06:46.655Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
147122024-02-04T14:06:46.655ZStates: Active Active Active
147132024-02-04T14:06:46.655ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147142024-02-04T14:06:46.655Z 1 Acked 1000 Flush 0 Done Done Sent false
147152024-02-04T14:06:46.655Z 2 Acked 1001 Write 8 Done Done Sent false
147162024-02-04T14:06:46.655Z 3 Acked 1002 Read 8 Done Done Sent false
147172024-02-04T14:06:46.655Z 4 Acked 1003 Write 3 Done Done Sent false
147182024-02-04T14:06:46.656Z 5 Acked 1004 Read 10 Done Done Sent false
147192024-02-04T14:06:46.656Z 6 Acked 1005 Read 10 Done Done Sent false
147202024-02-04T14:06:46.656Z 7 Acked 1006 Read 5 Done Done Sent false
147212024-02-04T14:06:46.656Z 8 Acked 1007 Flush 0 Done Done Sent false
147222024-02-04T14:06:46.656Z 9 Acked 1008 Write 4 Done Done Sent false
147232024-02-04T14:06:46.656Z 10 Acked 1009 Write 6 Done Done Sent false
147242024-02-04T14:06:46.656Z 11 Acked 1010 Read 10 Done Done Sent false
147252024-02-04T14:06:46.656Z 12 Acked 1011 Write 5 Done Done Sent false
147262024-02-04T14:06:46.656Z 13 Acked 1012 Read 1 Done Done Sent false
147272024-02-04T14:06:46.656Z 14 Acked 1013 Flush 0 Done Done Sent false
147282024-02-04T14:06:46.656Z 15 Acked 1014 Write 10 Done Done Sent false
147292024-02-04T14:06:46.656Z 16 Acked 1015 Read 1 Done Done Sent false
147302024-02-04T14:06:46.656Z 17 Acked 1016 Write 6 Done Done Sent false
147312024-02-04T14:06:46.656Z 18 Acked 1017 Read 3 Done Done Sent false
147322024-02-04T14:06:46.656Z 19 Acked 1018 Read 8 Done Done Sent false
147332024-02-04T14:06:46.656Z 20 Acked 1019 Flush 0 Done Done Sent false
147342024-02-04T14:06:46.656Z 21 Acked 1020 Write 7 Done Done Sent false
147352024-02-04T14:06:46.656Z 22 Acked 1021 Read 2 Done Done Sent false
147362024-02-04T14:06:46.656Z 23 Acked 1022 Read 2 Done Done Sent false
147372024-02-04T14:06:46.656Z 24 Acked 1023 Read 5 Done Done Sent false
147382024-02-04T14:06:46.656Z 25 Acked 1024 Read 2 Done Done Sent false
147392024-02-04T14:06:46.656Z 26 Acked 1025 Write 10 Done Done Sent false
147402024-02-04T14:06:46.656Z 27 Acked 1026 Write 10 Done Done Sent false
147412024-02-04T14:06:46.656Z 28 Acked 1027 Write 10 Done Done Sent false
147422024-02-04T14:06:46.657Z 29 Acked 1028 Read 6 Done Done Sent false
147432024-02-04T14:06:46.657Z 30 Acked 1029 Write 2 Sent Sent Sent false
147442024-02-04T14:06:46.657Z STATES DS:0 DS:1 DS:2 TOTAL
147452024-02-04T14:06:46.657Z New 0 0 0 0
147462024-02-04T14:06:46.657Z Sent 1 1 30 32
147472024-02-04T14:06:46.657Z Done 29 29 0 58
147482024-02-04T14:06:46.657Z Skipped 0 0 0 0
147492024-02-04T14:06:46.657Z Error 0 0 0 0
147502024-02-04T14:06:46.657ZLast Flush: 1019 1019 0
147512024-02-04T14:06:46.657ZDownstairs last five completed:
147522024-02-04T14:06:46.657ZUpstairs last five completed: 30 29 28 27 26
147532024-02-04T14:06:46.657Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:45.686302455Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1673}
147542024-02-04T14:06:46.657ZWrote out file "/tmp/test_repair_verify.data"
147552024-02-04T14:06:46.661Z
147562024-02-04T14:06:46.667ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147572024-02-04T14:06:46.694ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147582024-02-04T14:06:46.694Z 0 000-019 130 130 130 214 214 214 F F F
147592024-02-04T14:06:46.694Z 1 020-039 134 134 134 224 224 224 F F F
147602024-02-04T14:06:46.694Z 2 040-059 134 134 134 224 224 224 F F F
147612024-02-04T14:06:46.694Z 3 060-079 132 132 132 220 220 220 T T F <---
147622024-02-04T14:06:46.695Z 4 080-099 132 132 132 219 219 219 T T F <---
147632024-02-04T14:06:46.695Z 5 100-119 135 135 135 225 225 225 T T F <---
147642024-02-04T14:06:46.695Z 6 120-139 134 134 134 224 224 224 F F F
147652024-02-04T14:06:46.695Z 7 140-159 129 129 129 213 213 213 F F F
147662024-02-04T14:06:46.698Z 8 160-179 133 133 133 222 222 222 F F F
147672024-02-04T14:06:46.698Z 9 180-199 136 136 133 229 229 222 F F F <---
147682024-02-04T14:06:46.698Z 10 200-219 136 136 130 228 228 215 F F F <---
147692024-02-04T14:06:46.698Z 11 220-239 128 128 128 212 212 212 F F F
147702024-02-04T14:06:46.698Z 12 240-259 132 132 132 219 219 219 F F F
147712024-02-04T14:06:46.698Z 13 260-279 135 135 135 225 225 225 F F F
147722024-02-04T14:06:46.698Z 14 280-299 134 134 134 223 223 223 F F F
147732024-02-04T14:06:46.698Z 15 300-319 135 135 135 225 225 225 F F F
147742024-02-04T14:06:46.698Z 16 320-339 134 134 134 223 223 223 F F F
147752024-02-04T14:06:46.698Z 17 340-359 131 131 131 216 216 216 F F F
147762024-02-04T14:06:46.699Z 18 360-379 130 130 130 215 215 215 F F F
147772024-02-04T14:06:46.699Z 19 380-399 128 128 128 212 212 212 T T F <---
147782024-02-04T14:06:46.699Z 20 400-419 135 135 135 225 225 225 T T F <---
147792024-02-04T14:06:46.699Z 21 420-439 127 127 127 208 208 208 F F F
147802024-02-04T14:06:46.699Z 22 440-459 136 136 133 229 229 222 F F F <---
147812024-02-04T14:06:46.699Z 23 460-479 136 136 126 229 229 207 F F F <---
147822024-02-04T14:06:46.699Z 24 480-499 131 131 131 216 216 216 F F F
147832024-02-04T14:06:46.699Z 25 500-519 135 135 135 225 225 225 F F F
147842024-02-04T14:06:46.699Z 26 520-539 134 134 134 224 224 224 F F F
147852024-02-04T14:06:46.699Z 27 540-559 134 134 134 224 224 224 F F F
147862024-02-04T14:06:46.699Z 28 560-579 132 132 132 221 221 221 F F F
147872024-02-04T14:06:46.699Z 29 580-599 124 124 124 205 205 205 F F F
147882024-02-04T14:06:46.699ZMax gen: 136, Max flush: 229
147892024-02-04T14:06:46.699ZError: Difference in extent metadata found!
147902024-02-04T14:06:46.702ZA Difference in extent metadata is expected here
147912024-02-04T14:06:46.702ZOn loop 68
147922024-02-04T14:06:46.702Z
147932024-02-04T14:06:46.705ZVerifying data now
147942024-02-04T14:06:51.552ZNo change after verify
147952024-02-04T14:06:51.552ZLoop: 68 Downstairs dump after verify (and repair):
147962024-02-04T14:06:51.575ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147972024-02-04T14:06:51.575Z 0 000-019 130 130 130 214 214 214 F F F
147982024-02-04T14:06:51.575Z 1 020-039 134 134 134 224 224 224 F F F
147992024-02-04T14:06:51.575Z 2 040-059 134 134 134 224 224 224 F F F
148002024-02-04T14:06:51.575Z 3 060-079 137 137 137 230 230 230 F F F
148012024-02-04T14:06:51.575Z 4 080-099 137 137 137 230 230 230 F F F
148022024-02-04T14:06:51.575Z 5 100-119 137 137 137 230 230 230 F F F
148032024-02-04T14:06:51.575Z 6 120-139 134 134 134 224 224 224 F F F
148042024-02-04T14:06:51.579Z 7 140-159 129 129 129 213 213 213 F F F
148052024-02-04T14:06:51.579Z 8 160-179 133 133 133 222 222 222 F F F
148062024-02-04T14:06:51.579Z 9 180-199 136 136 136 229 229 229 F F F
148072024-02-04T14:06:51.579Z 10 200-219 136 136 136 228 228 228 F F F
148082024-02-04T14:06:51.579Z 11 220-239 128 128 128 212 212 212 F F F
148092024-02-04T14:06:51.579Z 12 240-259 132 132 132 219 219 219 F F F
148102024-02-04T14:06:51.579Z 13 260-279 135 135 135 225 225 225 F F F
148112024-02-04T14:06:51.579Z 14 280-299 134 134 134 223 223 223 F F F
148122024-02-04T14:06:51.579Z 15 300-319 135 135 135 225 225 225 F F F
148132024-02-04T14:06:51.579Z 16 320-339 134 134 134 223 223 223 F F F
148142024-02-04T14:06:51.579Z 17 340-359 131 131 131 216 216 216 F F F
148152024-02-04T14:06:51.579Z 18 360-379 130 130 130 215 215 215 F F F
148162024-02-04T14:06:51.579Z 19 380-399 137 137 137 230 230 230 F F F
148172024-02-04T14:06:51.579Z 20 400-419 137 137 137 230 230 230 F F F
148182024-02-04T14:06:51.579Z 21 420-439 127 127 127 208 208 208 F F F
148192024-02-04T14:06:51.579Z 22 440-459 136 136 136 229 229 229 F F F
148202024-02-04T14:06:51.579Z 23 460-479 136 136 136 229 229 229 F F F
148212024-02-04T14:06:51.579Z 24 480-499 131 131 131 216 216 216 F F F
148222024-02-04T14:06:51.579Z 25 500-519 135 135 135 225 225 225 F F F
148232024-02-04T14:06:51.580Z 26 520-539 134 134 134 224 224 224 F F F
148242024-02-04T14:06:51.580Z 27 540-559 134 134 134 224 224 224 F F F
148252024-02-04T14:06:51.580Z 28 560-579 132 132 132 221 221 221 F F F
148262024-02-04T14:06:51.580Z 29 580-599 124 124 124 205 205 205 F F F
148272024-02-04T14:06:51.580ZMax gen: 137, Max flush: 230
148282024-02-04T14:06:51.583Z
148292024-02-04T14:06:51.583ZBegin loop 69
148302024-02-04T14:06:51.583ZDownstairs to restart: 1
148312024-02-04T14:06:51.612Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.644100206Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681}
148322024-02-04T14:06:51.612Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.644571195Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681}
148332024-02-04T14:06:51.612Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.644643401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681}
148342024-02-04T14:06:51.612Z{Crucible runtime is spawned
148352024-02-04T14:06:51.615Z"msg":"Crucible 337823d3-0c06-466d-826b-63cdff638094 has session id: faba34dc-ef77-407e-bd9f-a548475af3f7","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.64475233Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148362024-02-04T14:06:51.616Z{"msg":"Upstairs opts: Upstairs UUID: 337823d3-0c06-466d-826b-63cdff638094, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.644843102Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148372024-02-04T14:06:51.616Z{"msg":"Crucible stats registered with UUID: 337823d3-0c06-466d-826b-63cdff638094","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.644888627Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148382024-02-04T14:06:51.616Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.644956192Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681}
148392024-02-04T14:06:51.616Z{"msg":"337823d3-0c06-466d-826b-63cdff638094 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645051358Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148402024-02-04T14:06:51.616Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645123749Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148412024-02-04T14:06:51.616Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645204538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148422024-02-04T14:06:51.616Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645249169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148432024-02-04T14:06:51.616Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645343629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148442024-02-04T14:06:51.616Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645386954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148452024-02-04T14:06:51.616Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645428496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148462024-02-04T14:06:51.616Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645473219Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148472024-02-04T14:06:51.616Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645553673Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148482024-02-04T14:06:51.616Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645626783Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148492024-02-04T14:06:51.616Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645701074Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148502024-02-04T14:06:51.616Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645798279Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148512024-02-04T14:06:51.616Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.645852037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148522024-02-04T14:06:51.616Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.647075879Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148532024-02-04T14:06:51.616Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.647133751Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148542024-02-04T14:06:51.616Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.64726986Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148552024-02-04T14:06:51.616Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.647329353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148562024-02-04T14:06:51.621Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.648087539Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148572024-02-04T14:06:51.621Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.648296742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148582024-02-04T14:06:51.621Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.648664476Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148592024-02-04T14:06:51.621Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.648784371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148602024-02-04T14:06:51.621Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.64899186Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148612024-02-04T14:06:51.621Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.649081724Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148622024-02-04T14:06:51.621Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.649159673Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148632024-02-04T14:06:51.621Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.649192197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148642024-02-04T14:06:51.621Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.649249794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148652024-02-04T14:06:51.621Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.649280782Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148662024-02-04T14:06:51.621Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.649657534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148672024-02-04T14:06:51.621Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650373777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148682024-02-04T14:06:51.621Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650434636Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148692024-02-04T14:06:51.621Z{"msg":"[0]R flush_numbers[0..12]: [214, 224, 224, 230, 230, 230, 224, 213, 222, 229, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650461185Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148702024-02-04T14:06:51.621Z{"msg":"[0]R generation[0..12]: [130, 134, 134, 137, 137, 137, 134, 129, 133, 136, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650504333Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148712024-02-04T14:06:51.621Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650530002Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148722024-02-04T14:06:51.621Z{"msg":"[1]R flush_numbers[0..12]: [214, 224, 224, 230, 230, 230, 224, 213, 222, 229, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650571886Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148732024-02-04T14:06:51.621Z{"msg":"[1]R generation[0..12]: [130, 134, 134, 137, 137, 137, 134, 129, 133, 136, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650596195Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148742024-02-04T14:06:51.621Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.6506386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148752024-02-04T14:06:51.621Z{"msg":"[2]R flush_numbers[0..12]: [214, 224, 224, 230, 230, 230, 224, 213, 222, 229, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650663907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148762024-02-04T14:06:51.621Z{"msg":"[2]R generation[0..12]: [130, 134, 134, 137, 137, 137, 134, 129, 133, 136, 136, 128]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
148772024-02-04T14:06:51.621Z,"level":30,"time":"2024-02-04T14:06:50.65070884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148782024-02-04T14:06:51.622Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650734469Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148792024-02-04T14:06:51.622Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650774848Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148802024-02-04T14:06:51.622Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650798335Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148812024-02-04T14:06:51.622Z{"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650838009Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148822024-02-04T14:06:51.622Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650860563Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148832024-02-04T14:06:51.622Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650900882Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148842024-02-04T14:06:51.622Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650921998Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148852024-02-04T14:06:51.622Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.65096301Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148862024-02-04T14:06:51.622Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.650985987Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148872024-02-04T14:06:51.622Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.651040952Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148882024-02-04T14:06:51.622Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.651091428Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148892024-02-04T14:06:51.622Z{"msg":"337823d3-0c06-466d-826b-63cdff638094 is now active with session: faba34dc-ef77-407e-bd9f-a548475af3f7","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.65111461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148902024-02-04T14:06:51.622Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.651139845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681,"session_id":"faba34dc-ef77-407e-bd9f-a548475af3f7"}
148912024-02-04T14:06:51.622Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:50.651196175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681}
148922024-02-04T14:06:51.622ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148932024-02-04T14:06:51.622ZLoading write count information from file "/tmp/test_repair_verify.data"
148942024-02-04T14:06:51.622ZRun Repair workload
148952024-02-04T14:06:51.622Z01/30 Write block 477 len 32768 data: 8 8 8 6 5 3 6 8
148962024-02-04T14:06:51.626Z02/30 Write block 445 len 24576 data: 13 14 14 13 12 12
148972024-02-04T14:06:51.626Z03/30 Write block 465 len 8192 data: 11 13
148982024-02-04T14:06:51.626Z04/30 Read block 472 len 32768
148992024-02-04T14:06:51.629Z05/30 Read block 587 len 4096
149002024-02-04T14:06:51.629Z06/30 Flush
149012024-02-04T14:06:51.632Z07/30 Write block 348 len 20480 data: 12 15 11 13 13
149022024-02-04T14:06:51.632Z08/30 Read block 533 len 24576
149032024-02-04T14:06:51.635Z09/30 Read block 336 len 16384
149042024-02-04T14:06:51.638Z10/30 Write block 5 len 40960 data: 9 10 10 9 11 7 6 4 4 5
149052024-02-04T14:06:51.639Z11/30 Write block 175 len 36864 data: 11 11 15 14 16 15 16 16 18
149062024-02-04T14:06:51.639Z12/30 Flush
149072024-02-04T14:06:51.642Z13/30 Flush
149082024-02-04T14:06:51.645Z14/30 Write block 570 len 20480 data: 11 10 9 9 8
149092024-02-04T14:06:51.645Z15/30 Read block 488 len 8192
149102024-02-04T14:06:51.659Z16/30 Write block 329 len 28672 data: 4 6 7 7 7 9 8
149112024-02-04T14:06:51.659Z17/30 Write block 171 len 36864 data: 9 13 13 12 12 12 16 15 17
149122024-02-04T14:06:51.659Z18/30 Read block 226 len 24576
149132024-02-04T14:06:51.663Z19/30 Read block 499 len 12288
149142024-02-04T14:06:51.663Z20/30 Write block 0 len 20480 data: 3 4 5 5 8
149152024-02-04T14:06:51.663Z21/30 Read block 308 len 16384
149162024-02-04T14:06:51.663Z22/30 Write block 251 len 4096 data: 15
149172024-02-04T14:06:51.663Z23/30 Read block 298 len 20480
149182024-02-04T14:06:51.663Z24/30 Write block 180 len 32768 data: 16 17 17 19 16 16 14 14
149192024-02-04T14:06:51.663Z25/30 Write block 18 len 36864 data: 11 9 9 9 12 13 12 10 10
149202024-02-04T14:06:51.663Z26/30 Write block 52 len 16384 data: 8 8 8 8
149212024-02-04T14:06:51.663Z27/30 Read block 386 len 4096
149222024-02-04T14:06:51.663Z28/30 Write block 256 len 12288 data: 12 10 10
149232024-02-04T14:06:51.663Z29/30 Read block 459 len 8192
149242024-02-04T14:06:51.667Z30/30 Read block 208 len 8192
149252024-02-04T14:06:51.667Z----------------------------------------------------------------
149262024-02-04T14:06:51.667Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
149272024-02-04T14:06:51.667ZStates: Active Active Active
149282024-02-04T14:06:51.667ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149292024-02-04T14:06:51.667Z 1 Acked 1000 Write 8 Done Done Done false
149302024-02-04T14:06:51.668Z 2 Acked 1001 Write 6 Done Done Done false
149312024-02-04T14:06:51.668Z 3 Acked 1002 Write 2 Done Done Done false
149322024-02-04T14:06:51.668Z 4 Acked 1003 Read 8 Done Done Done false
149332024-02-04T14:06:51.668Z 5 Acked 1004 Read 1 Done Sent Done false
149342024-02-04T14:06:51.668Z 6 Acked 1005 Flush 0 Done Sent Done false
149352024-02-04T14:06:51.668Z 7 Acked 1006 Write 5 Done Sent Done false
149362024-02-04T14:06:51.668Z 8 Acked 1007 Read 6 Done Sent Done false
149372024-02-04T14:06:51.668Z 9 Acked 1008 Read 4 Done Sent Done false
149382024-02-04T14:06:51.668Z 10 Acked 1009 Write 10 Done Sent Done false
149392024-02-04T14:06:51.668Z 11 Acked 1010 Write 9 Done Sent Done false
149402024-02-04T14:06:51.668Z 12 Acked 1011 Flush 0 Done Sent Done false
149412024-02-04T14:06:51.668Z 13 Acked 1012 Flush 0 Done Sent Done false
149422024-02-04T14:06:51.668Z 14 Acked 1013 Write 5 Done Sent Done false
149432024-02-04T14:06:51.668Z 15 Acked 1014 Read 2 Done Sent Done false
149442024-02-04T14:06:51.668Z 16 Acked 1015 Write 7 Done Sent Done false
149452024-02-04T14:06:51.669Z 17 Acked 1016 Write 9 Done Sent Done false
149462024-02-04T14:06:51.669Z 18 Acked 1017 Read 6 Done Sent Done false
149472024-02-04T14:06:51.669Z 19 Acked 1018 Read 3 Done Sent Done false
149482024-02-04T14:06:51.669Z 20 Acked 1019 Write 5 Done Sent Done false
149492024-02-04T14:06:51.669Z 21 Acked 1020 Read 4 Done Sent Done false
149502024-02-04T14:06:51.669Z 22 Acked 1021 Write 1 Done Sent Done false
149512024-02-04T14:06:51.669Z 23 Acked 1022 Read 5 Done Sent Done false
149522024-02-04T14:06:51.669Z 24 Acked 1023 Write 8 Done Sent Done false
149532024-02-04T14:06:51.669Z 25 Acked 1024 Write 9 Done Sent Done false
149542024-02-04T14:06:51.669Z 26 Acked 1025 Write 4 Done Sent Done false
149552024-02-04T14:06:51.669Z 27 Acked 1026 Read 1 Done Sent Done false
149562024-02-04T14:06:51.669Z 28 Acked 1027 Write 3 Done Sent Done false
149572024-02-04T14:06:51.669Z 29 Acked 1028 Read 2 Done Sent Done false
149582024-02-04T14:06:51.669Z 30 Acked 1029 Read 2 Done Sent Done false
149592024-02-04T14:06:51.669Z STATES DS:0 DS:1 DS:2 TOTAL
149602024-02-04T14:06:51.670Z New 0 0 0 0
149612024-02-04T14:06:51.670Z Sent 0 26 0 26
149622024-02-04T14:06:51.670Z Done 30 4 30 64
149632024-02-04T14:06:51.670Z Skipped 0 0 0 0
149642024-02-04T14:06:51.670Z Error 0 0 0 0
149652024-02-04T14:06:51.670ZLast Flush: 1012 0 1012
149662024-02-04T14:06:51.670ZDownstairs last five completed:
149672024-02-04T14:06:51.670ZUpstairs last five completed: 30 29 28 27 26
149682024-02-04T14:06:51.670Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:50.697743242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1681}
149692024-02-04T14:06:51.670ZWrote out file "/tmp/test_repair_verify.data"
149702024-02-04T14:06:51.674Z
149712024-02-04T14:06:51.679ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149722024-02-04T14:06:51.707ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149732024-02-04T14:06:51.707Z 0 000-019 138 130 138 232 214 232 T F T <---
149742024-02-04T14:06:51.707Z 1 020-039 134 134 134 224 224 224 T F T <---
149752024-02-04T14:06:51.707Z 2 040-059 134 134 134 224 224 224 T F T <---
149762024-02-04T14:06:51.708Z 3 060-079 137 137 137 230 230 230 F F F
149772024-02-04T14:06:51.708Z 4 080-099 137 137 137 230 230 230 F F F
149782024-02-04T14:06:51.708Z 5 100-119 137 137 137 230 230 230 F F F
149792024-02-04T14:06:51.708Z 6 120-139 134 134 134 224 224 224 F F F
149802024-02-04T14:06:51.712Z 7 140-159 129 129 129 213 213 213 F F F
149812024-02-04T14:06:51.712Z 8 160-179 138 133 138 232 222 232 T F T <---
149822024-02-04T14:06:51.712Z 9 180-199 138 136 138 232 229 232 T F T <---
149832024-02-04T14:06:51.712Z 10 200-219 136 136 136 228 228 228 F F F
149842024-02-04T14:06:51.712Z 11 220-239 128 128 128 212 212 212 F F F
149852024-02-04T14:06:51.712Z 12 240-259 132 132 132 219 219 219 T F T <---
149862024-02-04T14:06:51.712Z 13 260-279 135 135 135 225 225 225 F F F
149872024-02-04T14:06:51.712Z 14 280-299 134 134 134 223 223 223 F F F
149882024-02-04T14:06:51.713Z 15 300-319 135 135 135 225 225 225 F F F
149892024-02-04T14:06:51.713Z 16 320-339 134 134 134 223 223 223 T F T <---
149902024-02-04T14:06:51.713Z 17 340-359 138 131 138 232 216 232 F F F <---
149912024-02-04T14:06:51.713Z 18 360-379 130 130 130 215 215 215 F F F
149922024-02-04T14:06:51.713Z 19 380-399 137 137 137 230 230 230 F F F
149932024-02-04T14:06:51.713Z 20 400-419 137 137 137 230 230 230 F F F
149942024-02-04T14:06:51.713Z 21 420-439 127 127 127 208 208 208 F F F
149952024-02-04T14:06:51.713Z 22 440-459 138 136 138 231 229 231 F T F <---
149962024-02-04T14:06:51.713Z 23 460-479 138 136 138 231 229 231 F T F <---
149972024-02-04T14:06:51.713Z 24 480-499 138 131 138 231 216 231 F T F <---
149982024-02-04T14:06:51.713Z 25 500-519 135 135 135 225 225 225 F F F
149992024-02-04T14:06:51.713Z 26 520-539 134 134 134 224 224 224 F F F
150002024-02-04T14:06:51.713Z 27 540-559 134 134 134 224 224 224 F F F
150012024-02-04T14:06:51.713Z 28 560-579 132 132 132 221 221 221 T F T <---
150022024-02-04T14:06:51.713Z 29 580-599 124 124 124 205 205 205 F F F
150032024-02-04T14:06:51.713ZMax gen: 138, Max flush: 232
150042024-02-04T14:06:51.714ZError: Difference in extent metadata found!
150052024-02-04T14:06:51.717ZA Difference in extent metadata is expected here
150062024-02-04T14:06:51.717ZOn loop 69
150072024-02-04T14:06:51.717Z
150082024-02-04T14:06:51.720ZVerifying data now
150092024-02-04T14:06:57.777ZNo change after verify
150102024-02-04T14:06:57.777ZLoop: 69 Downstairs dump after verify (and repair):
150112024-02-04T14:06:57.800ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150122024-02-04T14:06:57.800Z 0 000-019 139 139 139 233 233 233 F F F
150132024-02-04T14:06:57.800Z 1 020-039 139 139 139 233 233 233 F F F
150142024-02-04T14:06:57.800Z 2 040-059 139 139 139 233 233 233 F F F
150152024-02-04T14:06:57.800Z 3 060-079 137 137 137 230 230 230 F F F
150162024-02-04T14:06:57.800Z 4 080-099 137 137 137 230 230 230 F F F
150172024-02-04T14:06:57.803Z 5 100-119 137 137 137 230 230 230 F F F
150182024-02-04T14:06:57.803Z 6 120-139 134 134 134 224 224 224 F F F
150192024-02-04T14:06:57.803Z 7 140-159 129 129 129 213 213 213 F F F
150202024-02-04T14:06:57.803Z 8 160-179 139 139 139 233 233 233 F F F
150212024-02-04T14:06:57.803Z 9 180-199 139 139 139 233 233 233 F F F
150222024-02-04T14:06:57.803Z 10 200-219 136 136 136 228 228 228 F F F
150232024-02-04T14:06:57.803Z 11 220-239 128 128 128 212 212 212 F F F
150242024-02-04T14:06:57.803Z 12 240-259 139 139 139 233 233 233 F F F
150252024-02-04T14:06:57.803Z 13 260-279 135 135 135 225 225 225 F F F
150262024-02-04T14:06:57.803Z 14 280-299 134 134 134 223 223 223 F F F
150272024-02-04T14:06:57.803Z 15 300-319 135 135 135 225 225 225 F F F
150282024-02-04T14:06:57.803Z 16 320-339 139 139 139 233 233 233 F F F
150292024-02-04T14:06:57.803Z 17 340-359 138 138 138 232 232 232 F F F
150302024-02-04T14:06:57.804Z 18 360-379 130 130 130 215 215 215 F F F
150312024-02-04T14:06:57.804Z 19 380-399 137 137 137 230 230 230 F F F
150322024-02-04T14:06:57.804Z 20 400-419 137 137 137 230 230 230 F F F
150332024-02-04T14:06:57.804Z 21 420-439 127 127 127 208 208 208 F F F
150342024-02-04T14:06:57.804Z 22 440-459 138 138 138 231 231 231 F F F
150352024-02-04T14:06:57.804Z 23 460-479 138 138 138 231 231 231 F F F
150362024-02-04T14:06:57.804Z 24 480-499 138 138 138 231 231 231 F F F
150372024-02-04T14:06:57.804Z 25 500-519 135 135 135 225 225 225 F F F
150382024-02-04T14:06:57.804Z 26 520-539 134 134 134 224 224 224 F F F
150392024-02-04T14:06:57.804Z 27 540-559 134 134 134 224 224 224 F F F
150402024-02-04T14:06:57.804Z 28 560-579 139 139 139 233 233 233 F F F
150412024-02-04T14:06:57.804Z 29 580-599 124 124 124 205 205 205 F F F
150422024-02-04T14:06:57.804ZMax gen: 139, Max flush: 233
150432024-02-04T14:06:57.807Z
150442024-02-04T14:06:57.807ZBegin loop 70
150452024-02-04T14:06:57.807ZDownstairs to restart: 2
150462024-02-04T14:06:57.834Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.866495961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689}
150472024-02-04T14:06:57.834Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.866980306Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689}
150482024-02-04T14:06:57.840Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867045247Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689}
150492024-02-04T14:06:57.840Z{"msg":"Crucible 3bfb74de-7e81-4396-84c2-780167d4106f has session id: 22a915ad-e8f7-45b5-8a1d-052337819a4b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867140749Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150502024-02-04T14:06:57.840Z{"msg":"Crucible runtime is spawned
150512024-02-04T14:06:57.840ZUpstairs opts: Upstairs UUID: 3bfb74de-7e81-4396-84c2-780167d4106f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867387932Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150522024-02-04T14:06:57.840Z{"msg":"Crucible stats registered with UUID: 3bfb74de-7e81-4396-84c2-780167d4106f","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867446365Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150532024-02-04T14:06:57.840Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867493409Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689}
150542024-02-04T14:06:57.840Z{"msg":"3bfb74de-7e81-4396-84c2-780167d4106f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867645408Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150552024-02-04T14:06:57.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867708969Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150562024-02-04T14:06:57.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867755416Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150572024-02-04T14:06:57.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867797901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150582024-02-04T14:06:57.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.86784041Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150592024-02-04T14:06:57.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867928157Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150602024-02-04T14:06:57.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.867973107Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150612024-02-04T14:06:57.840Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.868062888Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150622024-02-04T14:06:57.841Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.868198741Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150632024-02-04T14:06:57.841Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.868330703Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150642024-02-04T14:06:57.841Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.868367718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150652024-02-04T14:06:57.841Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.868391806Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150662024-02-04T14:06:57.841Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.868471402Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150672024-02-04T14:06:57.841Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.869624267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150682024-02-04T14:06:57.841Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.869704927Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150692024-02-04T14:06:57.841Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.869812445Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150702024-02-04T14:06:57.845Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.869896651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150712024-02-04T14:06:57.845Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.8703481Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150722024-02-04T14:06:57.845Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.870418727Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150732024-02-04T14:06:57.845Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.870626907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150742024-02-04T14:06:57.845Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.870976219Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150752024-02-04T14:06:57.845Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.871570586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150762024-02-04T14:06:57.845Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.871858995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150772024-02-04T14:06:57.845Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.871981898Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150782024-02-04T14:06:57.846Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872053538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150792024-02-04T14:06:57.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872078642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150802024-02-04T14:06:57.846Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872135693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150812024-02-04T14:06:57.846Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872160025Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150822024-02-04T14:06:57.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872367093Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150832024-02-04T14:06:57.846Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.87243311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150842024-02-04T14:06:57.846Z{"msg":"[0]R flush_numbers[0..12]: [233, 233, 233, 230, 230, 230, 224, 213, 233, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872479606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150852024-02-04T14:06:57.846Z{"msg":"[0]R generation[0..12]: [139, 139, 139, 137, 137, 137, 134, 129, 139, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872536726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150862024-02-04T14:06:57.846Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
150872024-02-04T14:06:57.846Z,"time":"2024-02-04T14:06:56.872582392Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150882024-02-04T14:06:57.846Z{"msg":"[1]R flush_numbers[0..12]: [233, 233, 233, 230, 230, 230, 224, 213, 233, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872680725Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150892024-02-04T14:06:57.846Z{"msg":"[1]R generation[0..12]: [139, 139, 139, 137, 137, 137, 134, 129, 139, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872749705Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150902024-02-04T14:06:57.846Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872826728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150912024-02-04T14:06:57.846Z{"msg":"[2]R flush_numbers[0..12]: [233, 233, 233, 230, 230, 230, 224, 213, 233, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872870636Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150922024-02-04T14:06:57.846Z{"msg":"[2]R generation[0..12]: [139, 139, 139, 137, 137, 137, 134, 129, 139, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872925542Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150932024-02-04T14:06:57.846Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.872970225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150942024-02-04T14:06:57.846Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873024375Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150952024-02-04T14:06:57.846Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873065403Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150962024-02-04T14:06:57.846Z{"msg":"Next flush: 234","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873125154Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150972024-02-04T14:06:57.846ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150982024-02-04T14:06:57.846Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873175662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
150992024-02-04T14:06:57.846Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873229726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
151002024-02-04T14:06:57.846Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.87326803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
151012024-02-04T14:06:57.846Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873322268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
151022024-02-04T14:06:57.847Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873364304Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
151032024-02-04T14:06:57.847Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873416393Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
151042024-02-04T14:06:57.847Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873455488Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
151052024-02-04T14:06:57.847Z{"msg":"3bfb74de-7e81-4396-84c2-780167d4106f is now active with session: 22a915ad-e8f7-45b5-8a1d-052337819a4b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873508772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
151062024-02-04T14:06:57.847Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873546707Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689,"session_id":"22a915ad-e8f7-45b5-8a1d-052337819a4b"}
151072024-02-04T14:06:57.847Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:06:56.873597281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689}
151082024-02-04T14:06:57.847ZLoading write count information from file "/tmp/test_repair_verify.data"
151092024-02-04T14:06:57.847ZRun Repair workload
151102024-02-04T14:06:57.847Z01/30 Flush
151112024-02-04T14:06:57.847Z02/30 Write block 247 len 12288 data: 12 16 15
151122024-02-04T14:06:57.847Z03/30 Read block 106 len 36864
151132024-02-04T14:06:57.850Z04/30 Flush
151142024-02-04T14:06:57.850Z05/30 Write block 422 len 8192 data: 11 11
151152024-02-04T14:06:57.850Z06/30 Write block 358 len 4096 data: 10
151162024-02-04T14:06:57.850Z07/30 Read block 355 len 8192
151172024-02-04T14:06:57.854Z08/30 Read block 551 len 8192
151182024-02-04T14:06:57.854Z09/30 Write block 326 len 40960 data: 6 6 5 5 7 8 8 8 10 9
151192024-02-04T14:06:57.857Z10/30 Flush
151202024-02-04T14:06:57.878Z11/30 Write block 166 len 8192 data: 12 12
151212024-02-04T14:06:57.878Z12/30 Read block 351 len 28672
151222024-02-04T14:06:57.878Z13/30 Read block 130 len 20480
151232024-02-04T14:06:57.878Z14/30 Read block 99 len 16384
151242024-02-04T14:06:57.878Z15/30 Flush
151252024-02-04T14:06:57.878Z16/30 Write block 522 len 36864 data: 9 9 11 10 13 12 12 10 9
151262024-02-04T14:06:57.878Z17/30 Read block 581 len 36864
151272024-02-04T14:06:57.878Z18/30 Read block 583 len 8192
151282024-02-04T14:06:57.878Z19/30 Flush
151292024-02-04T14:06:57.879Z20/30 Write block 158 len 40960 data: 7 8 8 7 8 11 10 12 13 13
151302024-02-04T14:06:57.879Z21/30 Read block 80 len 28672
151312024-02-04T14:06:57.879Z22/30 Write block 508 len 20480 data: 15 15 13 11 6
151322024-02-04T14:06:57.879Z23/30 Write block 524 len 24576 data: 12 11 14 13 13 11
151332024-02-04T14:06:57.879Z24/30 Write block 360 len 32768 data: 9 10 11 11 10 9 10 12
151342024-02-04T14:06:57.883Z25/30 Write block 110 len 32768 data: 13 14 17 14 16 16 13 13
151352024-02-04T14:06:57.883Z26/30 Write block 40 len 16384 data: 8 8 10 9
151362024-02-04T14:06:57.883Z27/30 Write block 161 len 28672 data: 8 9 12 11 13 14 14
151372024-02-04T14:06:57.883Z28/30 Read block 200 len 12288
151382024-02-04T14:06:57.887Z29/30 Write block 539 len 24576 data: 8 9 9 8 11 9
151392024-02-04T14:06:57.887Z30/30 Write block 407 len 32768 data: 14 12 13 12 12 13 13 13
151402024-02-04T14:06:57.887Z----------------------------------------------------------------
151412024-02-04T14:06:57.887Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:29
151422024-02-04T14:06:57.887ZStates: Active Active Active
151432024-02-04T14:06:57.887ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151442024-02-04T14:06:57.887Z 2 Acked 1001 Write 3 Done Done Done false
151452024-02-04T14:06:57.887Z 3 Acked 1002 Read 9 Done Done Sent false
151462024-02-04T14:06:57.887Z 4 Acked 1003 Flush 0 Done Done Sent false
151472024-02-04T14:06:57.887Z 5 Acked 1004 Write 2 Done Done Sent false
151482024-02-04T14:06:57.887Z 6 Acked 1005 Write 1 Done Done Sent false
151492024-02-04T14:06:57.887Z 7 Acked 1006 Read 2 Done Done Sent false
151502024-02-04T14:06:57.887Z 8 Acked 1007 Read 2 Done Done Sent false
151512024-02-04T14:06:57.887Z 9 Acked 1008 Write 10 Done Done Sent false
151522024-02-04T14:06:57.888Z 10 Acked 1009 Flush 0 Done Done Sent false
151532024-02-04T14:06:57.888Z 11 Acked 1010 Write 2 Done Done Sent false
151542024-02-04T14:06:57.888Z 12 Acked 1011 Read 7 Done Done Sent false
151552024-02-04T14:06:57.888Z 13 Acked 1012 Read 5 Done Done Sent false
151562024-02-04T14:06:57.888Z 14 Acked 1013 Read 4 Done Done Sent false
151572024-02-04T14:06:57.888Z 15 Acked 1014 Flush 0 Done Done Sent false
151582024-02-04T14:06:57.888Z 16 Acked 1015 Write 9 Done Done Sent false
151592024-02-04T14:06:57.888Z 17 Acked 1016 Read 9 Done Done Sent false
151602024-02-04T14:06:57.888Z 18 Acked 1017 Read 2 Done Done Sent false
151612024-02-04T14:06:57.888Z 19 Acked 1018 Flush 0 Done Done Sent false
151622024-02-04T14:06:57.888Z 20 Acked 1019 Write 10 Done Done Sent false
151632024-02-04T14:06:57.888Z 21 Acked 1020 Read 7 Done Done Sent false
151642024-02-04T14:06:57.888Z 22 Acked 1021 Write 5 Done Done Sent false
151652024-02-04T14:06:57.888Z 23 Acked 1022 Write 6 Done Done Sent false
151662024-02-04T14:06:57.888Z 24 Acked 1023 Write 8 Done Done Sent false
151672024-02-04T14:06:57.888Z 25 Acked 1024 Write 8 Done Done Sent false
151682024-02-04T14:06:57.888Z 26 Acked 1025 Write 4 Done Done Sent false
151692024-02-04T14:06:57.888Z 27 Acked 1026 Write 7 Done Done Sent false
151702024-02-04T14:06:57.888Z 28 Acked 1027 Read 3 Done Done Sent false
151712024-02-04T14:06:57.888Z 29 Acked 1028 Write 6 Done Done Sent false
151722024-02-04T14:06:57.888Z 30 Acked 1029 Write 8 Sent Sent Sent false
151732024-02-04T14:06:57.888Z STATES DS:0 DS:1 DS:2 TOTAL
151742024-02-04T14:06:57.888Z New 0 0 0 0
151752024-02-04T14:06:57.889Z Sent 1 1 28 30
151762024-02-04T14:06:57.889Z Done 28 28 1 57
151772024-02-04T14:06:57.889Z Skipped 0 0 0 0
151782024-02-04T14:06:57.889Z Error 0 0 0 0
151792024-02-04T14:06:57.889ZLast Flush: 1018 1018 1000
151802024-02-04T14:06:57.889ZDownstairs last five completed: 1000
151812024-02-04T14:06:57.889ZUpstairs last five completed: 30 29 28 27 26
151822024-02-04T14:06:57.892Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:06:56.919710596Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1689}
151832024-02-04T14:06:57.892ZWrote out file "/tmp/test_repair_verify.data"
151842024-02-04T14:06:57.895Z
151852024-02-04T14:06:57.899ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151862024-02-04T14:06:57.926ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151872024-02-04T14:06:57.927Z 0 000-019 139 139 139 233 233 233 F F F
151882024-02-04T14:06:57.927Z 1 020-039 139 139 139 233 233 233 F F F
151892024-02-04T14:06:57.927Z 2 040-059 139 139 139 233 233 233 T T F <---
151902024-02-04T14:06:57.930Z 3 060-079 137 137 137 230 230 230 F F F
151912024-02-04T14:06:57.930Z 4 080-099 137 137 137 230 230 230 F F F
151922024-02-04T14:06:57.930Z 5 100-119 137 137 137 230 230 230 T T F <---
151932024-02-04T14:06:57.930Z 6 120-139 134 134 134 224 224 224 F F F
151942024-02-04T14:06:57.930Z 7 140-159 129 129 129 213 213 213 T T F <---
151952024-02-04T14:06:57.930Z 8 160-179 140 140 139 237 237 233 T T F <---
151962024-02-04T14:06:57.930Z 9 180-199 139 139 139 233 233 233 F F F
151972024-02-04T14:06:57.930Z 10 200-219 136 136 136 228 228 228 F F F
151982024-02-04T14:06:57.930Z 11 220-239 128 128 128 212 212 212 F F F
151992024-02-04T14:06:57.930Z 12 240-259 140 140 139 235 235 233 F F T <---
152002024-02-04T14:06:57.931Z 13 260-279 135 135 135 225 225 225 F F F
152012024-02-04T14:06:57.931Z 14 280-299 134 134 134 223 223 223 F F F
152022024-02-04T14:06:57.931Z 15 300-319 135 135 135 225 225 225 F F F
152032024-02-04T14:06:57.931Z 16 320-339 140 140 139 236 236 233 F F F <---
152042024-02-04T14:06:57.931Z 17 340-359 140 140 138 236 236 232 F F F <---
152052024-02-04T14:06:57.931Z 18 360-379 130 130 130 215 215 215 T T F <---
152062024-02-04T14:06:57.931Z 19 380-399 137 137 137 230 230 230 F F F
152072024-02-04T14:06:57.931Z 20 400-419 137 137 137 230 230 230 T T F <---
152082024-02-04T14:06:57.931Z 21 420-439 140 140 127 236 236 208 F F F <---
152092024-02-04T14:06:57.931Z 22 440-459 138 138 138 231 231 231 F F F
152102024-02-04T14:06:57.931Z 23 460-479 138 138 138 231 231 231 F F F
152112024-02-04T14:06:57.931Z 24 480-499 138 138 138 231 231 231 F F F
152122024-02-04T14:06:57.931Z 25 500-519 135 135 135 225 225 225 T T F <---
152132024-02-04T14:06:57.931Z 26 520-539 140 140 134 238 238 224 T T F <---
152142024-02-04T14:06:57.931Z 27 540-559 134 134 134 224 224 224 T T F <---
152152024-02-04T14:06:57.931Z 28 560-579 139 139 139 233 233 233 F F F
152162024-02-04T14:06:57.931Z 29 580-599 124 124 124 205 205 205 F F F
152172024-02-04T14:06:57.931ZMax gen: 140, Max flush: 238
152182024-02-04T14:06:57.931ZError: Difference in extent metadata found!
152192024-02-04T14:06:57.934ZA Difference in extent metadata is expected here
152202024-02-04T14:06:57.934ZOn loop 70
152212024-02-04T14:06:57.934Z
152222024-02-04T14:06:57.934ZVerifying data now
152232024-02-04T14:07:03.066ZNo change after verify
152242024-02-04T14:07:03.066ZLoop: 70 Downstairs dump after verify (and repair):
152252024-02-04T14:07:03.089ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152262024-02-04T14:07:03.089Z 0 000-019 139 139 139 233 233 233 F F F
152272024-02-04T14:07:03.089Z 1 020-039 139 139 139 233 233 233 F F F
152282024-02-04T14:07:03.089Z 2 040-059 141 141 141 239 239 239 F F F
152292024-02-04T14:07:03.089Z 3 060-079 137 137 137 230 230 230 F F F
152302024-02-04T14:07:03.089Z 4 080-099 137 137 137 230 230 230 F F F
152312024-02-04T14:07:03.089Z 5 100-119 141 141 141 239 239 239 F F F
152322024-02-04T14:07:03.089Z 6 120-139 134 134 134 224 224 224 F F F
152332024-02-04T14:07:03.089Z 7 140-159 141 141 141 239 239 239 F F F
152342024-02-04T14:07:03.089Z 8 160-179 141 141 141 239 239 239 F F F
152352024-02-04T14:07:03.089Z 9 180-199 139 139 139 233 233 233 F F F
152362024-02-04T14:07:03.093Z 10 200-219 136 136 136 228 228 228 F F F
152372024-02-04T14:07:03.093Z 11 220-239 128 128 128 212 212 212 F F F
152382024-02-04T14:07:03.093Z 12 240-259 140 140 140 235 235 235 F F F
152392024-02-04T14:07:03.094Z 13 260-279 135 135 135 225 225 225 F F F
152402024-02-04T14:07:03.094Z 14 280-299 134 134 134 223 223 223 F F F
152412024-02-04T14:07:03.094Z 15 300-319 135 135 135 225 225 225 F F F
152422024-02-04T14:07:03.094Z 16 320-339 140 140 140 236 236 236 F F F
152432024-02-04T14:07:03.094Z 17 340-359 140 140 140 236 236 236 F F F
152442024-02-04T14:07:03.094Z 18 360-379 141 141 141 239 239 239 F F F
152452024-02-04T14:07:03.094Z 19 380-399 137 137 137 230 230 230 F F F
152462024-02-04T14:07:03.094Z 20 400-419 141 141 141 239 239 239 F F F
152472024-02-04T14:07:03.094Z 21 420-439 140 140 140 236 236 236 F F F
152482024-02-04T14:07:03.094Z 22 440-459 138 138 138 231 231 231 F F F
152492024-02-04T14:07:03.094Z 23 460-479 138 138 138 231 231 231 F F F
152502024-02-04T14:07:03.094Z 24 480-499 138 138 138 231 231 231 F F F
152512024-02-04T14:07:03.094Z 25 500-519 141 141 141 239 239 239 F F F
152522024-02-04T14:07:03.094Z 26 520-539 141 141 141 239 239 239 F F F
152532024-02-04T14:07:03.094Z 27 540-559 141 141 141 239 239 239 F F F
152542024-02-04T14:07:03.094Z 28 560-579 139 139 139 233 233 233 F F F
152552024-02-04T14:07:03.094Z 29 580-599 124 124 124 205 205 205 F F F
152562024-02-04T14:07:03.094ZMax gen: 141, Max flush: 239
152572024-02-04T14:07:03.098Z
152582024-02-04T14:07:03.098ZBegin loop 71
152592024-02-04T14:07:03.098ZDownstairs to restart: 1
152602024-02-04T14:07:03.130Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.161775263Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697}
152612024-02-04T14:07:03.130Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.162225699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697}
152622024-02-04T14:07:03.130Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.162253831Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697}
152632024-02-04T14:07:03.130Z{"msg":"Crucible 5a5be33f-6518-4563-8434-d3bbeca67dc3 has session id: 32fc4dff-ace5-4eeb-b184-982fe5b964e0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.162274951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152642024-02-04T14:07:03.134Z{"msg":"Upstairs opts: Upstairs UUID: 5a5be33f-6518-4563-8434-d3bbeca67dc3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.162345411Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152652024-02-04T14:07:03.134Z{"msg":"Crucible stats registered with UUID: 5a5be33f-6518-4563-8434-d3bbeca67dc3","v":0,"name":"crucible","level":30Crucible runtime is spawned
152662024-02-04T14:07:03.134Z,"time":"2024-02-04T14:07:02.16241719Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152672024-02-04T14:07:03.134Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.16255594Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697}
152682024-02-04T14:07:03.134Z{"msg":"5a5be33f-6518-4563-8434-d3bbeca67dc3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.162804556Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152692024-02-04T14:07:03.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.16286211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152702024-02-04T14:07:03.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.162887746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152712024-02-04T14:07:03.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.162952186Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152722024-02-04T14:07:03.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.163030923Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152732024-02-04T14:07:03.134Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.163073551Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152742024-02-04T14:07:03.134Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.163178898Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152752024-02-04T14:07:03.135Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.16327319Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152762024-02-04T14:07:03.135Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.163320795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152772024-02-04T14:07:03.135Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.163366453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152782024-02-04T14:07:03.135Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.163407037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152792024-02-04T14:07:03.135Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.163452175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152802024-02-04T14:07:03.135Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.163553979Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152812024-02-04T14:07:03.135Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.16483323Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152822024-02-04T14:07:03.135Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.164967816Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152832024-02-04T14:07:03.135Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.165044801Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152842024-02-04T14:07:03.135Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.165096861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152852024-02-04T14:07:03.135Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.165961125Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152862024-02-04T14:07:03.135Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.16617901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152872024-02-04T14:07:03.140Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.166872436Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152882024-02-04T14:07:03.140Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.166957571Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152892024-02-04T14:07:03.140Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.167149677Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152902024-02-04T14:07:03.140Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.16721072Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152912024-02-04T14:07:03.140Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.167236675Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152922024-02-04T14:07:03.140Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.167282669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152932024-02-04T14:07:03.140Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.167412901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152942024-02-04T14:07:03.140Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.167487061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152952024-02-04T14:07:03.140Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.168135404Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152962024-02-04T14:07:03.140Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.168855842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152972024-02-04T14:07:03.140Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.168937246Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152982024-02-04T14:07:03.140Z{"msg":"[0]R flush_numbers[0..12]: [233, 233, 239, 230, 230, 239, 224, 239, 239, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.16896901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
152992024-02-04T14:07:03.140Z{"msg":"[0]R generation[0..12]: [139, 139, 141, 137, 137, 141, 134, 141, 141, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169028831Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153002024-02-04T14:07:03.141Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169073044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153012024-02-04T14:07:03.141Z{"msg":"[1]R flush_numbers[0..12]: [233, 233, 239, 230, 230, 239, 224, 239, 239, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169139485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153022024-02-04T14:07:03.141Z{"msg":"[1]R generation[0..12]: [139, 139, 141, 137, 137, 141, 134, 141, 141, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169220248Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153032024-02-04T14:07:03.141ZWait for a query_work_queue command to finish before sending IO
153042024-02-04T14:07:03.141Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169272408Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153052024-02-04T14:07:03.141Z{"msg":"[2]R flush_numbers[0..12]: [233, 233, 239, 230, 230, 239, 224, 239, 239, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169333797Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153062024-02-04T14:07:03.141Z{"msg":"[2]R generation[0..12]: [139, 139, 141, 137, 137, 141, 134, 141, 141, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169379061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153072024-02-04T14:07:03.141Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169439422Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153082024-02-04T14:07:03.141Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169483743Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153092024-02-04T14:07:03.141Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169544166Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153102024-02-04T14:07:03.141Z{"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169586688Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153112024-02-04T14:07:03.141Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169652956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153122024-02-04T14:07:03.141Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169696298Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153132024-02-04T14:07:03.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169755334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153142024-02-04T14:07:03.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169797939Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153152024-02-04T14:07:03.141Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169857827Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153162024-02-04T14:07:03.141Z32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153172024-02-04T14:07:03.141Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169913004Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153182024-02-04T14:07:03.141Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.169966332Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153192024-02-04T14:07:03.141Z{"msg":"5a5be33f-6518-4563-8434-d3bbeca67dc3 is now active with session: 32fc4dff-ace5-4eeb-b184-982fe5b964e0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.170012899Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153202024-02-04T14:07:03.141Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.17006006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697,"session_id":"32fc4dff-ace5-4eeb-b184-982fe5b964e0"}
153212024-02-04T14:07:03.141Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:02.170083074Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697}
153222024-02-04T14:07:03.141ZLoading write count information from file "/tmp/test_repair_verify.data"
153232024-02-04T14:07:03.142ZRun Repair workload
153242024-02-04T14:07:03.142Z01/30 Write block 516 len 32768 data: 11 10 9 9 5 7 10 10
153252024-02-04T14:07:03.145Z02/30 Read block 486 len 16384
153262024-02-04T14:07:03.145Z03/30 Read block 530 len 28672
153272024-02-04T14:07:03.146Z04/30 Read block 350 len 4096
153282024-02-04T14:07:03.146Z05/30 Flush
153292024-02-04T14:07:03.149Z06/30 Write block 404 len 8192 data: 12 15
153302024-02-04T14:07:03.152Z07/30 Write block 499 len 16384 data: 15 15 15 12
153312024-02-04T14:07:03.152Z08/30 Read block 110 len 40960
153322024-02-04T14:07:03.155Z09/30 Read block 250 len 40960
153332024-02-04T14:07:03.155Z10/30 Read block 488 len 28672
153342024-02-04T14:07:03.158Z11/30 Write block 82 len 12288 data: 16 14 14
153352024-02-04T14:07:03.158Z12/30 Flush
153362024-02-04T14:07:03.161Z13/30 Write block 258 len 12288 data: 11 11 10
153372024-02-04T14:07:03.162Z14/30 Read block 120 len 32768
153382024-02-04T14:07:03.165Z15/30 Write block 319 len 32768 data: 10 9 8 8 7 6 6 7
153392024-02-04T14:07:03.165Z16/30 Read block 448 len 40960
153402024-02-04T14:07:03.168Z17/30 Read block 113 len 24576
153412024-02-04T14:07:03.168Z18/30 Read block 115 len 20480
153422024-02-04T14:07:03.172Z19/30 Write block 297 len 16384 data: 6 8 9 8
153432024-02-04T14:07:03.172Z20/30 Read block 108 len 12288
153442024-02-04T14:07:03.172Z21/30 Flush
153452024-02-04T14:07:03.175Z22/30 Write block 540 len 36864 data: 10 10 9 12 10 9 7 7 7
153462024-02-04T14:07:03.178Z23/30 Read block 593 len 24576
153472024-02-04T14:07:03.178Z24/30 Read block 49 len 28672
153482024-02-04T14:07:03.181Z25/30 Flush
153492024-02-04T14:07:03.185Z26/30 Write block 130 len 12288 data: 7 6 8
153502024-02-04T14:07:03.185Z27/30 Read block 237 len 16384
153512024-02-04T14:07:03.185Z28/30 Write block 116 len 28672 data: 14 14 11 11 11 13 12
153522024-02-04T14:07:03.189Z29/30 Read block 591 len 32768
153532024-02-04T14:07:03.193Z30/30 Write block 538 len 4096 data: 7
153542024-02-04T14:07:03.193Z----------------------------------------------------------------
153552024-02-04T14:07:03.193Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:18
153562024-02-04T14:07:03.193ZStates: Active Active Active
153572024-02-04T14:07:03.193ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153582024-02-04T14:07:03.193Z 13 Acked 1012 Write 3 Done Done Done false
153592024-02-04T14:07:03.193Z 14 Acked 1013 Read 8 Done Sent Done false
153602024-02-04T14:07:03.193Z 15 Acked 1014 Write 8 Done Sent Done false
153612024-02-04T14:07:03.193Z 16 Acked 1015 Read 10 Done Sent Done false
153622024-02-04T14:07:03.193Z 17 Acked 1016 Read 6 Done Sent Done false
153632024-02-04T14:07:03.193Z 18 Acked 1017 Read 5 Done Sent Done false
153642024-02-04T14:07:03.193Z 19 Acked 1018 Write 4 Done Sent Done false
153652024-02-04T14:07:03.193Z 20 Acked 1019 Read 3 Done Sent Done false
153662024-02-04T14:07:03.194Z 21 Acked 1020 Flush 0 Done Sent Done false
153672024-02-04T14:07:03.194Z 22 Acked 1021 Write 9 Done Sent Done false
153682024-02-04T14:07:03.194Z 23 Acked 1022 Read 6 Done Sent Done false
153692024-02-04T14:07:03.194Z 24 Acked 1023 Read 7 Done Sent Done false
153702024-02-04T14:07:03.194Z 25 Acked 1024 Flush 0 Done Sent Done false
153712024-02-04T14:07:03.194Z 26 Acked 1025 Write 3 Done Sent Done false
153722024-02-04T14:07:03.194Z 27 Acked 1026 Read 4 Done Sent Done false
153732024-02-04T14:07:03.194Z 28 Acked 1027 Write 7 Done Sent Done false
153742024-02-04T14:07:03.194Z 29 Acked 1028 Read 8 Done Sent Done false
153752024-02-04T14:07:03.194Z 30 Acked 1029 Write 1 Sent Sent Sent false
153762024-02-04T14:07:03.194Z STATES DS:0 DS:1 DS:2 TOTAL
153772024-02-04T14:07:03.194Z New 0 0 0 0
153782024-02-04T14:07:03.194Z Sent 1 17 1 19
153792024-02-04T14:07:03.194Z Done 17 1 17 35
153802024-02-04T14:07:03.194Z Skipped 0 0 0 0
153812024-02-04T14:07:03.194Z Error 0 0 0 0
153822024-02-04T14:07:03.194ZLast Flush: 1024 1011 1024
153832024-02-04T14:07:03.194ZDownstairs last five completed: 1011 1010 1009 1008 1007
153842024-02-04T14:07:03.195ZUpstairs last five completed: 30 29 28 27 26
153852024-02-04T14:07:03.195Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:02.222691293Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1697}
153862024-02-04T14:07:03.195ZWrote out file "/tmp/test_repair_verify.data"
153872024-02-04T14:07:03.202Z
153882024-02-04T14:07:03.207ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153892024-02-04T14:07:03.232ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153902024-02-04T14:07:03.232Z 0 000-019 139 139 139 233 233 233 F F F
153912024-02-04T14:07:03.232Z 1 020-039 139 139 139 233 233 233 F F F
153922024-02-04T14:07:03.232Z 2 040-059 141 141 141 239 239 239 F F F
153932024-02-04T14:07:03.232Z 3 060-079 137 137 137 230 230 230 F F F
153942024-02-04T14:07:03.233Z 4 080-099 142 142 142 241 241 241 F F F
153952024-02-04T14:07:03.233Z 5 100-119 141 141 141 239 239 239 T F T <---
153962024-02-04T14:07:03.233Z 6 120-139 134 134 134 224 224 224 T F T <---
153972024-02-04T14:07:03.233Z 7 140-159 141 141 141 239 239 239 F F F
153982024-02-04T14:07:03.233Z 8 160-179 141 141 141 239 239 239 F F F
153992024-02-04T14:07:03.233Z 9 180-199 139 139 139 233 233 233 F F F
154002024-02-04T14:07:03.233Z 10 200-219 136 136 136 228 228 228 F F F
154012024-02-04T14:07:03.233Z 11 220-239 128 128 128 212 212 212 F F F
154022024-02-04T14:07:03.237Z 12 240-259 142 140 142 242 235 242 F T F <---
154032024-02-04T14:07:03.238Z 13 260-279 142 135 142 242 225 242 F T F <---
154042024-02-04T14:07:03.238Z 14 280-299 142 134 142 242 223 242 F F F <---
154052024-02-04T14:07:03.238Z 15 300-319 142 135 142 242 225 242 F F F <---
154062024-02-04T14:07:03.238Z 16 320-339 142 140 142 242 236 242 F F F <---
154072024-02-04T14:07:03.238Z 17 340-359 140 140 140 236 236 236 F F F
154082024-02-04T14:07:03.241Z 18 360-379 141 141 141 239 239 239 F F F
154092024-02-04T14:07:03.242Z 19 380-399 137 137 137 230 230 230 F F F
154102024-02-04T14:07:03.242Z 20 400-419 142 142 142 241 241 241 F F F
154112024-02-04T14:07:03.242Z 21 420-439 140 140 140 236 236 236 F F F
154122024-02-04T14:07:03.242Z 22 440-459 138 138 138 231 231 231 F F F
154132024-02-04T14:07:03.242Z 23 460-479 138 138 138 231 231 231 F F F
154142024-02-04T14:07:03.242Z 24 480-499 142 142 142 241 241 241 F F F
154152024-02-04T14:07:03.242Z 25 500-519 142 142 142 241 241 241 F F F
154162024-02-04T14:07:03.242Z 26 520-539 142 142 142 240 240 240 T F T <---
154172024-02-04T14:07:03.242Z 27 540-559 142 141 142 243 239 243 F F F <---
154182024-02-04T14:07:03.242Z 28 560-579 139 139 139 233 233 233 F F F
154192024-02-04T14:07:03.242Z 29 580-599 124 124 124 205 205 205 F F F
154202024-02-04T14:07:03.242ZMax gen: 142, Max flush: 243
154212024-02-04T14:07:03.243ZError: Difference in extent metadata found!
154222024-02-04T14:07:03.243ZA Difference in extent metadata is expected here
154232024-02-04T14:07:03.243ZOn loop 71
154242024-02-04T14:07:03.243Z
154252024-02-04T14:07:03.247ZVerifying data now
154262024-02-04T14:07:09.080ZNo change after verify
154272024-02-04T14:07:09.080ZLoop: 71 Downstairs dump after verify (and repair):
154282024-02-04T14:07:09.104ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154292024-02-04T14:07:09.104Z 0 000-019 139 139 139 233 233 233 F F F
154302024-02-04T14:07:09.104Z 1 020-039 139 139 139 233 233 233 F F F
154312024-02-04T14:07:09.104Z 2 040-059 141 141 141 239 239 239 F F F
154322024-02-04T14:07:09.111Z 3 060-079 137 137 137 230 230 230 F F F
154332024-02-04T14:07:09.111Z 4 080-099 142 142 142 241 241 241 F F F
154342024-02-04T14:07:09.111Z 5 100-119 143 143 143 244 244 244 F F F
154352024-02-04T14:07:09.112Z 6 120-139 143 143 143 244 244 244 F F F
154362024-02-04T14:07:09.112Z 7 140-159 141 141 141 239 239 239 F F F
154372024-02-04T14:07:09.112Z 8 160-179 141 141 141 239 239 239 F F F
154382024-02-04T14:07:09.112Z 9 180-199 139 139 139 233 233 233 F F F
154392024-02-04T14:07:09.112Z 10 200-219 136 136 136 228 228 228 F F F
154402024-02-04T14:07:09.112Z 11 220-239 128 128 128 212 212 212 F F F
154412024-02-04T14:07:09.112Z 12 240-259 142 142 142 242 242 242 F F F
154422024-02-04T14:07:09.112Z 13 260-279 142 142 142 242 242 242 F F F
154432024-02-04T14:07:09.112Z 14 280-299 142 142 142 242 242 242 F F F
154442024-02-04T14:07:09.112Z 15 300-319 142 142 142 242 242 242 F F F
154452024-02-04T14:07:09.112Z 16 320-339 142 142 142 242 242 242 F F F
154462024-02-04T14:07:09.112Z 17 340-359 140 140 140 236 236 236 F F F
154472024-02-04T14:07:09.112Z 18 360-379 141 141 141 239 239 239 F F F
154482024-02-04T14:07:09.112Z 19 380-399 137 137 137 230 230 230 F F F
154492024-02-04T14:07:09.112Z 20 400-419 142 142 142 241 241 241 F F F
154502024-02-04T14:07:09.112Z 21 420-439 140 140 140 236 236 236 F F F
154512024-02-04T14:07:09.112Z 22 440-459 138 138 138 231 231 231 F F F
154522024-02-04T14:07:09.112Z 23 460-479 138 138 138 231 231 231 F F F
154532024-02-04T14:07:09.112Z 24 480-499 142 142 142 241 241 241 F F F
154542024-02-04T14:07:09.112Z 25 500-519 142 142 142 241 241 241 F F F
154552024-02-04T14:07:09.112Z 26 520-539 143 143 143 244 244 244 F F F
154562024-02-04T14:07:09.112Z 27 540-559 142 142 142 243 243 243 F F F
154572024-02-04T14:07:09.112Z 28 560-579 139 139 139 233 233 233 F F F
154582024-02-04T14:07:09.112Z 29 580-599 124 124 124 205 205 205 F F F
154592024-02-04T14:07:09.113ZMax gen: 143, Max flush: 244
154602024-02-04T14:07:09.113Z
154612024-02-04T14:07:09.113ZBegin loop 72
154622024-02-04T14:07:09.113ZDownstairs to restart: 0
154632024-02-04T14:07:09.139Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.17128451Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705}
154642024-02-04T14:07:09.146Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.171897005Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705}
154652024-02-04T14:07:09.146Z{Crucible runtime is spawned
154662024-02-04T14:07:09.146Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172036199Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705}
154672024-02-04T14:07:09.146Z{"msg":"Crucible a4a46e5a-5600-4677-9660-ace40438f8e1 has session id: bda91ee3-8f24-48a4-b245-9d0b27c7cec2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.17213096Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154682024-02-04T14:07:09.146Z{"msg":"Upstairs opts: Upstairs UUID: a4a46e5a-5600-4677-9660-ace40438f8e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172180189Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154692024-02-04T14:07:09.146Z{"msg":"Crucible stats registered with UUID: a4a46e5a-5600-4677-9660-ace40438f8e1","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172206115Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154702024-02-04T14:07:09.146Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172247658Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705}
154712024-02-04T14:07:09.146Z{"msg":"a4a46e5a-5600-4677-9660-ace40438f8e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172410234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154722024-02-04T14:07:09.146Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172467174Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154732024-02-04T14:07:09.146Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172492772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154742024-02-04T14:07:09.146Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172516982Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154752024-02-04T14:07:09.146Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172540268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154762024-02-04T14:07:09.146Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172562743Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154772024-02-04T14:07:09.146Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172586998Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154782024-02-04T14:07:09.147Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172662025Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"io task","client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154792024-02-04T14:07:09.147Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.172898569Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"io task","client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154802024-02-04T14:07:09.147Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.173140609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"io task","client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154812024-02-04T14:07:09.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.173203687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"io task","client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154822024-02-04T14:07:09.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.173229859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"io task","client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154832024-02-04T14:07:09.147Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.173282216Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"io task","client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154842024-02-04T14:07:09.147Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.174132954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154852024-02-04T14:07:09.147Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.174266693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154862024-02-04T14:07:09.147Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.174397032Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154872024-02-04T14:07:09.147Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.174463231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154882024-02-04T14:07:09.147Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.175014149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154892024-02-04T14:07:09.147Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.175178504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154902024-02-04T14:07:09.147Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.175792965Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154912024-02-04T14:07:09.147Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.17591011Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154922024-02-04T14:07:09.147Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.176018782Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154932024-02-04T14:07:09.147Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.17610563Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154942024-02-04T14:07:09.147Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.176176971Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154952024-02-04T14:07:09.147Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.176333056Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154962024-02-04T14:07:09.147Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.176394019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154972024-02-04T14:07:09.147Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.176465766Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154982024-02-04T14:07:09.156Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.176847684Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
154992024-02-04T14:07:09.156Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.177551919Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155002024-02-04T14:07:09.156Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.17770491Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155012024-02-04T14:07:09.156Z{"msg":"[0]R flush_numbers[0..12]: [233, 233, 239, 230, 241, 244, 244, 239, 239, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.177755907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155022024-02-04T14:07:09.156Z{"msg":"[0]R generation[0..12]: [139, 139, 141, 137, 142, 143, 143, 141, 141, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.177781601Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155032024-02-04T14:07:09.156Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO
155042024-02-04T14:07:09.156Z","level":30,"time":"2024-02-04T14:07:08.177849719Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155052024-02-04T14:07:09.156Z{"msg":"[1]R flush_numbers[0..12]: [233, 233, 239, 230, 241, 244, 244, 239, 239, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.177921929Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155062024-02-04T14:07:09.156Z{"msg":"[1]R generation[0..12]: [139, 139, 141, 137, 142, 143, 143, 141, 141, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.177974312Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155072024-02-04T14:07:09.156Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.17801549Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155082024-02-04T14:07:09.156Z{"msg":"[2]R flush_numbers[0..12]: [233, 233, 239, 230, 241, 244, 244, 239, 239, 233, 228, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178078459Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155092024-02-04T14:07:09.156Z{"msg":"[2]R generation[0..12]: [139, 139, 141, 137, 142, 143, 143, 141, 141, 139, 136, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178147964Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155102024-02-04T14:07:09.156Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178225956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155112024-02-04T14:07:09.156Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178293861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155122024-02-04T14:07:09.156Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178356501Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155132024-02-04T14:07:09.156Z{"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178437404Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155142024-02-04T14:07:09.156Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178480877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155152024-02-04T14:07:09.156Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178522549Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155162024-02-04T14:07:09.156Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178556864Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155172024-02-04T14:07:09.157Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178626285Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155182024-02-04T14:07:09.157Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155192024-02-04T14:07:09.157Z,"time":"2024-02-04T14:07:08.178651833Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155202024-02-04T14:07:09.157Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178763869Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155212024-02-04T14:07:09.157Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178789714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155222024-02-04T14:07:09.157Z{"msg":"a4a46e5a-5600-4677-9660-ace40438f8e1 is now active with session: bda91ee3-8f24-48a4-b245-9d0b27c7cec2","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178812478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155232024-02-04T14:07:09.157Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178852272Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705,"session_id":"bda91ee3-8f24-48a4-b245-9d0b27c7cec2"}
155242024-02-04T14:07:09.157Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:08.178874232Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705}
155252024-02-04T14:07:09.157ZLoading write count information from file "/tmp/test_repair_verify.data"
155262024-02-04T14:07:09.157ZRun Repair workload
155272024-02-04T14:07:09.157Z01/30 Write block 393 len 24576 data: 15 13 13 11 14 12
155282024-02-04T14:07:09.157Z02/30 Write block 265 len 28672 data: 11 12 9 12 11 10 9
155292024-02-04T14:07:09.157Z03/30 Read block 207 len 16384
155302024-02-04T14:07:09.162Z04/30 Flush
155312024-02-04T14:07:09.162Z05/30 Write block 33 len 40960 data: 10 10 13 14 12 10 9 9 9 11
155322024-02-04T14:07:09.162Z06/30 Read block 157 len 28672
155332024-02-04T14:07:09.162Z07/30 Read block 229 len 40960
155342024-02-04T14:07:09.165Z08/30 Read block 528 len 28672
155352024-02-04T14:07:09.165Z09/30 Read block 453 len 32768
155362024-02-04T14:07:09.165Z10/30 Write block 524 len 36864 data: 13 12 15 14 14 12 10 7 7
155372024-02-04T14:07:09.169Z11/30 Read block 549 len 20480
155382024-02-04T14:07:09.169Z12/30 Flush
155392024-02-04T14:07:09.174Z13/30 Write block 100 len 20480 data: 7 8 8 7 8
155402024-02-04T14:07:09.174Z14/30 Read block 349 len 8192
155412024-02-04T14:07:09.178Z15/30 Write block 241 len 40960 data: 13 11 10 11 10 12 13 17 16 16
155422024-02-04T14:07:09.178Z16/30 Write block 237 len 16384 data: 14 12 11 14
155432024-02-04T14:07:09.178Z17/30 Write block 363 len 24576 data: 12 11 10 11 13 11
155442024-02-04T14:07:09.181Z18/30 Write block 333 len 12288 data: 9 11 10
155452024-02-04T14:07:09.181Z19/30 Read block 536 len 40960
155462024-02-04T14:07:09.181Z20/30 Read block 535 len 20480
155472024-02-04T14:07:09.185Z21/30 Read block 442 len 12288
155482024-02-04T14:07:09.185Z22/30 Read block 87 len 8192
155492024-02-04T14:07:09.185Z23/30 Read block 31 len 8192
155502024-02-04T14:07:09.189Z24/30 Write block 556 len 40960 data: 13 14 12 14 15 12 11 13 15 16
155512024-02-04T14:07:09.189Z25/30 Flush
155522024-02-04T14:07:09.195Z26/30 Write block 410 len 16384 data: 13 13 14 14
155532024-02-04T14:07:09.196Z27/30 Read block 313 len 36864
155542024-02-04T14:07:09.196Z28/30 Read block 128 len 8192
155552024-02-04T14:07:09.201Z29/30 Write block 345 len 20480 data: 11 10 11 13 16
155562024-02-04T14:07:09.201Z30/30 Write block 434 len 16384 data: 13 13 12 10
155572024-02-04T14:07:09.202Z----------------------------------------------------------------
155582024-02-04T14:07:09.202Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
155592024-02-04T14:07:09.202ZStates: Active Active Active
155602024-02-04T14:07:09.202ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155612024-02-04T14:07:09.202Z 1 Acked 1000 Write 6 Done Done Done false
155622024-02-04T14:07:09.202Z 2 Acked 1001 Write 7 Done Done Done false
155632024-02-04T14:07:09.202Z 3 Acked 1002 Read 4 Sent Done Done false
155642024-02-04T14:07:09.202Z 4 Acked 1003 Flush 0 Sent Done Done false
155652024-02-04T14:07:09.202Z 5 Acked 1004 Write 10 Sent Done Done false
155662024-02-04T14:07:09.202Z 6 Acked 1005 Read 7 Sent Done Done false
155672024-02-04T14:07:09.202Z 7 Acked 1006 Read 10 Sent Done Done false
155682024-02-04T14:07:09.202Z 8 Acked 1007 Read 7 Sent Done Done false
155692024-02-04T14:07:09.202Z 9 Acked 1008 Read 8 Sent Done Done false
155702024-02-04T14:07:09.202Z 10 Acked 1009 Write 9 Sent Done Done false
155712024-02-04T14:07:09.202Z 11 Acked 1010 Read 5 Sent Done Done false
155722024-02-04T14:07:09.202Z 12 Acked 1011 Flush 0 Sent Done Done false
155732024-02-04T14:07:09.202Z 13 Acked 1012 Write 5 Sent Done Done false
155742024-02-04T14:07:09.202Z 14 Acked 1013 Read 2 Sent Done Done false
155752024-02-04T14:07:09.202Z 15 Acked 1014 Write 10 Sent Done Done false
155762024-02-04T14:07:09.202Z 16 Acked 1015 Write 4 Sent Done Done false
155772024-02-04T14:07:09.202Z 17 Acked 1016 Write 6 Sent Done Done false
155782024-02-04T14:07:09.203Z 18 Acked 1017 Write 3 Sent Done Done false
155792024-02-04T14:07:09.203Z 19 Acked 1018 Read 10 Sent Done Done false
155802024-02-04T14:07:09.203Z 20 Acked 1019 Read 5 Sent Done Done false
155812024-02-04T14:07:09.203Z 21 Acked 1020 Read 3 Sent Done Done false
155822024-02-04T14:07:09.203Z 22 Acked 1021 Read 2 Sent Done Done false
155832024-02-04T14:07:09.203Z 23 Acked 1022 Read 2 Sent Done Done false
155842024-02-04T14:07:09.203Z 24 Acked 1023 Write 10 Sent Done Done false
155852024-02-04T14:07:09.203Z 25 Acked 1024 Flush 0 Sent Done Done false
155862024-02-04T14:07:09.203Z 26 Acked 1025 Write 4 Sent Done Done false
155872024-02-04T14:07:09.203Z 27 Acked 1026 Read 9 Sent Done Done false
155882024-02-04T14:07:09.203Z 28 Acked 1027 Read 2 Sent Done Done false
155892024-02-04T14:07:09.203Z 29 Acked 1028 Write 5 Sent Done Done false
155902024-02-04T14:07:09.203Z 30 Acked 1029 Write 4 Sent Sent Sent false
155912024-02-04T14:07:09.203Z STATES DS:0 DS:1 DS:2 TOTAL
155922024-02-04T14:07:09.203Z New 0 0 0 0
155932024-02-04T14:07:09.203Z Sent 28 1 1 30
155942024-02-04T14:07:09.203Z Done 2 29 29 60
155952024-02-04T14:07:09.203Z Skipped 0 0 0 0
155962024-02-04T14:07:09.203Z Error 0 0 0 0
155972024-02-04T14:07:09.203ZLast Flush: 0 1024 1024
155982024-02-04T14:07:09.203ZDownstairs last five completed:
155992024-02-04T14:07:09.203ZUpstairs last five completed: 30 29 28 27 26
156002024-02-04T14:07:09.203Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:08.230885005Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1705}
156012024-02-04T14:07:09.203ZWrote out file "/tmp/test_repair_verify.data"
156022024-02-04T14:07:09.207Z
156032024-02-04T14:07:09.213ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
156042024-02-04T14:07:09.237ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156052024-02-04T14:07:09.237Z 0 000-019 139 139 139 233 233 233 F F F
156062024-02-04T14:07:09.237Z 1 020-039 139 144 144 233 246 246 F F F <---
156072024-02-04T14:07:09.237Z 2 040-059 141 144 144 239 246 246 F F F <---
156082024-02-04T14:07:09.237Z 3 060-079 137 137 137 230 230 230 F F F
156092024-02-04T14:07:09.238Z 4 080-099 142 142 142 241 241 241 F F F
156102024-02-04T14:07:09.238Z 5 100-119 143 144 144 244 247 247 F F F <---
156112024-02-04T14:07:09.238Z 6 120-139 143 143 143 244 244 244 F F F
156122024-02-04T14:07:09.238Z 7 140-159 141 141 141 239 239 239 F F F
156132024-02-04T14:07:09.238Z 8 160-179 141 141 141 239 239 239 F F F
156142024-02-04T14:07:09.238Z 9 180-199 139 139 139 233 233 233 F F F
156152024-02-04T14:07:09.238Z 10 200-219 136 136 136 228 228 228 F F F
156162024-02-04T14:07:09.238Z 11 220-239 128 144 144 212 247 247 F F F <---
156172024-02-04T14:07:09.243Z 12 240-259 142 144 144 242 247 247 F F F <---
156182024-02-04T14:07:09.243Z 13 260-279 142 144 144 242 245 245 T F F <---
156192024-02-04T14:07:09.243Z 14 280-299 142 142 142 242 242 242 F F F
156202024-02-04T14:07:09.243Z 15 300-319 142 142 142 242 242 242 F F F
156212024-02-04T14:07:09.243Z 16 320-339 142 144 144 242 247 247 F F F <---
156222024-02-04T14:07:09.243Z 17 340-359 140 140 140 236 236 236 F T T <---
156232024-02-04T14:07:09.243Z 18 360-379 141 144 144 239 247 247 F F F <---
156242024-02-04T14:07:09.243Z 19 380-399 137 144 144 230 245 245 T F F <---
156252024-02-04T14:07:09.243Z 20 400-419 142 142 142 241 241 241 F T T <---
156262024-02-04T14:07:09.243Z 21 420-439 140 140 140 236 236 236 F T T <---
156272024-02-04T14:07:09.243Z 22 440-459 138 138 138 231 231 231 F F F
156282024-02-04T14:07:09.243Z 23 460-479 138 138 138 231 231 231 F F F
156292024-02-04T14:07:09.243Z 24 480-499 142 142 142 241 241 241 F F F
156302024-02-04T14:07:09.243Z 25 500-519 142 142 142 241 241 241 F F F
156312024-02-04T14:07:09.243Z 26 520-539 143 144 144 244 246 246 F F F <---
156322024-02-04T14:07:09.243Z 27 540-559 142 144 144 243 247 247 F F F <---
156332024-02-04T14:07:09.244Z 28 560-579 139 144 144 233 247 247 F F F <---
156342024-02-04T14:07:09.244Z 29 580-599 124 124 124 205 205 205 F F F
156352024-02-04T14:07:09.244ZMax gen: 144, Max flush: 247
156362024-02-04T14:07:09.244ZError: Difference in extent metadata found!
156372024-02-04T14:07:09.244ZA Difference in extent metadata is expected here
156382024-02-04T14:07:09.244ZOn loop 72
156392024-02-04T14:07:09.244Z
156402024-02-04T14:07:09.252ZVerifying data now
156412024-02-04T14:07:15.504ZNo change after verify
156422024-02-04T14:07:15.505ZLoop: 72 Downstairs dump after verify (and repair):
156432024-02-04T14:07:15.926ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156442024-02-04T14:07:15.926Z 0 000-019 139 139 139 233 233 233 F F F
156452024-02-04T14:07:15.927Z 1 020-039 144 144 144 246 246 246 F F F
156462024-02-04T14:07:15.927Z 2 040-059 144 144 144 246 246 246 F F F
156472024-02-04T14:07:15.927Z 3 060-079 137 137 137 230 230 230 F F F
156482024-02-04T14:07:15.962Z 4 080-099 142 142 142 241 241 241 F F F
156492024-02-04T14:07:15.962Z 5 100-119 144 144 144 247 247 247 F F F
156502024-02-04T14:07:15.962Z 6 120-139 143 143 143 244 244 244 F F F
156512024-02-04T14:07:15.962Z 7 140-159 141 141 141 239 239 239 F F F
156522024-02-04T14:07:15.962Z 8 160-179 141 141 141 239 239 239 F F F
156532024-02-04T14:07:15.962Z 9 180-199 139 139 139 233 233 233 F F F
156542024-02-04T14:07:15.963Z 10 200-219 136 136 136 228 228 228 F F F
156552024-02-04T14:07:15.963Z 11 220-239 144 144 144 247 247 247 F F F
156562024-02-04T14:07:15.963Z 12 240-259 144 144 144 247 247 247 F F F
156572024-02-04T14:07:15.963Z 13 260-279 144 144 144 245 245 245 F F F
156582024-02-04T14:07:15.963Z 14 280-299 142 142 142 242 242 242 F F F
156592024-02-04T14:07:15.964Z 15 300-319 142 142 142 242 242 242 F F F
156602024-02-04T14:07:15.964Z 16 320-339 144 144 144 247 247 247 F F F
156612024-02-04T14:07:15.964Z 17 340-359 145 145 145 248 248 248 F F F
156622024-02-04T14:07:15.964Z 18 360-379 144 144 144 247 247 247 F F F
156632024-02-04T14:07:15.964Z 19 380-399 144 144 144 245 245 245 F F F
156642024-02-04T14:07:15.964Z 20 400-419 145 145 145 248 248 248 F F F
156652024-02-04T14:07:15.964Z 21 420-439 145 145 145 248 248 248 F F F
156662024-02-04T14:07:15.965Z 22 440-459 138 138 138 231 231 231 F F F
156672024-02-04T14:07:15.965Z 23 460-479 138 138 138 231 231 231 F F F
156682024-02-04T14:07:15.965Z 24 480-499 142 142 142 241 241 241 F F F
156692024-02-04T14:07:15.965Z 25 500-519 142 142 142 241 241 241 F F F
156702024-02-04T14:07:15.965Z 26 520-539 144 144 144 246 246 246 F F F
156712024-02-04T14:07:15.965Z 27 540-559 144 144 144 247 247 247 F F F
156722024-02-04T14:07:15.965Z 28 560-579 144 144 144 247 247 247 F F F
156732024-02-04T14:07:15.965Z 29 580-599 124 124 124 205 205 205 F F F
156742024-02-04T14:07:15.966ZMax gen: 145, Max flush: 248
156752024-02-04T14:07:15.966Z
156762024-02-04T14:07:15.966ZBegin loop 73
156772024-02-04T14:07:15.966ZDownstairs to restart: 1
156782024-02-04T14:07:15.966Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.58500013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713}
156792024-02-04T14:07:15.966Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.585467079Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713}
156802024-02-04T14:07:15.966Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.585519401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713}
156812024-02-04T14:07:15.967Z{"msg":"Crucible 995a51a7-9059-415c-a49f-6bccd1fba4fe has session id: 1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.585586631Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156822024-02-04T14:07:15.967Z{"msg":"Crucible runtime is spawned
156832024-02-04T14:07:15.967ZUpstairs opts: Upstairs UUID: 995a51a7-9059-415c-a49f-6bccd1fba4fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.585691621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156842024-02-04T14:07:15.967Z{"msg":"Crucible stats registered with UUID: 995a51a7-9059-415c-a49f-6bccd1fba4fe","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.585753268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156852024-02-04T14:07:15.967Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.585777679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713}
156862024-02-04T14:07:15.967Z{"msg":"995a51a7-9059-415c-a49f-6bccd1fba4fe active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.585986766Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156872024-02-04T14:07:15.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586041023Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156882024-02-04T14:07:15.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586066699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156892024-02-04T14:07:15.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586165643Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156902024-02-04T14:07:15.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586344647Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156912024-02-04T14:07:15.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.58646629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156922024-02-04T14:07:15.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586629044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156932024-02-04T14:07:15.968Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586688878Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"io task","client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156942024-02-04T14:07:15.968Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586751031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"io task","client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156952024-02-04T14:07:15.968Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586779974Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"io task","client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156962024-02-04T14:07:15.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586837341Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"io task","client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156972024-02-04T14:07:15.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586862066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"io task","client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156982024-02-04T14:07:15.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.586891574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"io task","client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
156992024-02-04T14:07:15.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.588361938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157002024-02-04T14:07:15.969Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.588479015Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157012024-02-04T14:07:15.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.58861197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157022024-02-04T14:07:15.969Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.588653031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157032024-02-04T14:07:15.969Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.589385324Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157042024-02-04T14:07:15.969Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.589595644Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157052024-02-04T14:07:15.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.589779383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157062024-02-04T14:07:15.970Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.589841999Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157072024-02-04T14:07:15.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.590245617Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157082024-02-04T14:07:15.970Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.590332602Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157092024-02-04T14:07:15.970Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.590383454Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157102024-02-04T14:07:15.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.590458477Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157112024-02-04T14:07:15.970Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.59053854Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157122024-02-04T14:07:15.970Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.590612081Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157132024-02-04T14:07:15.971Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.590832055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157142024-02-04T14:07:15.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591522386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157152024-02-04T14:07:15.971Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591586043Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157162024-02-04T14:07:15.971Z{"msg":"[0]R flush_numbers[0..12]: [233, 246, 246, 230, 241, 247, 244, 239, 239, 233, 228, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591655082Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157172024-02-04T14:07:15.971Z{"msg":"[0]R generation[0..12]: [139, 144, 144, 137, 142, 144, 143, 141, 141, 139, 136, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591700166Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157182024-02-04T14:07:15.971Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591726454Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157192024-02-04T14:07:15.971Z{"msg":"[1]R flush_numbers[0..12]: [233, 246, 246, 230, 241, 247, 244, 239, 239, 233, 228, 247]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
157202024-02-04T14:07:15.972Z,"time":"2024-02-04T14:07:14.591769346Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157212024-02-04T14:07:15.972Z{"msg":"[1]R generation[0..12]: [139, 144, 144, 137, 142, 144, 143, 141, 141, 139, 136, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591794892Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157222024-02-04T14:07:15.972Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591837807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157232024-02-04T14:07:15.972Z{"msg":"[2]R flush_numbers[0..12]: [233, 246, 246, 230, 241, 247, 244, 239, 239, 233, 228, 247]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591863613Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157242024-02-04T14:07:15.972Z{"msg":"[2]R generation[0..12]: [139, 144, 144, 137, 142, 144, 143, 141, 141, 139, 136, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591909876Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157252024-02-04T14:07:15.972Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.59193516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157262024-02-04T14:07:15.972Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.591976643Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157272024-02-04T14:07:15.973Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592000335Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157282024-02-04T14:07:15.973Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592042845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157292024-02-04T14:07:15.973Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592065813Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157302024-02-04T14:07:15.973Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592090068Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157312024-02-04T14:07:15.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592110732Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157322024-02-04T14:07:15.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592133241Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157332024-02-04T14:07:15.973Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592181725Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157342024-02-04T14:07:15.973Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.59220571Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157352024-02-04T14:07:15.973Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592226794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157362024-02-04T14:07:15.973Z{"msg":"995a51a7-9059-415c-a49f-6bccd1fba4fe is now active with session: 1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592273744Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157372024-02-04T14:07:15.973Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.59229547Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713,"session_id":"1cbf43c9-cbf6-4208-9acd-3c8a15d2c06c"}
157382024-02-04T14:07:15.974Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:14.592337341Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713}
157392024-02-04T14:07:15.974ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157402024-02-04T14:07:15.974ZLoading write count information from file "/tmp/test_repair_verify.data"
157412024-02-04T14:07:15.974ZRun Repair workload
157422024-02-04T14:07:15.974Z01/30 Write block 539 len 4096 data: 9
157432024-02-04T14:07:15.974Z02/30 Read block 243 len 16384
157442024-02-04T14:07:15.974Z03/30 Read block 127 len 20480
157452024-02-04T14:07:15.974Z04/30 Read block 514 len 20480
157462024-02-04T14:07:15.974Z05/30 Read block 452 len 16384
157472024-02-04T14:07:15.975Z06/30 Read block 400 len 36864
157482024-02-04T14:07:15.999Z07/30 Flush
157492024-02-04T14:07:15.999Z08/30 Write block 467 len 40960 data: 12 10 11 10 8 9 9 8 10 9
157502024-02-04T14:07:16.000Z09/30 Write block 549 len 36864 data: 8 8 6 6 10 11 11 14 15
157512024-02-04T14:07:16.000Z10/30 Read block 347 len 4096
157522024-02-04T14:07:16.001Z11/30 Read block 503 len 4096
157532024-02-04T14:07:16.001Z12/30 Write block 464 len 24576 data: 9 12 14 13 11 12
157542024-02-04T14:07:16.001Z13/30 Read block 367 len 16384
157552024-02-04T14:07:16.001Z14/30 Write block 234 len 16384 data: 13 10 11 15
157562024-02-04T14:07:16.002Z15/30 Flush
157572024-02-04T14:07:16.002Z16/30 Write block 441 len 32768 data: 9 10 12 13 14 15 15 14
157582024-02-04T14:07:16.002Z17/30 Write block 88 len 36864 data: 7 4 4 6 6 5 6 9 8
157592024-02-04T14:07:16.002Z18/30 Read block 536 len 16384
157602024-02-04T14:07:16.002Z19/30 Flush
157612024-02-04T14:07:16.003Z20/30 Write block 16 len 36864 data: 7 8 12 10 10 10 13 14 13
157622024-02-04T14:07:16.003Z21/30 Write block 124 len 16384 data: 10 12 12 11
157632024-02-04T14:07:16.003Z22/30 Read block 462 len 24576
157642024-02-04T14:07:16.003Z23/30 Write block 258 len 4096 data: 12
157652024-02-04T14:07:16.004Z24/30 Read block 427 len 20480
157662024-02-04T14:07:16.004Z25/30 Read block 431 len 32768
157672024-02-04T14:07:16.004Z26/30 Write block 528 len 4096 data: 15
157682024-02-04T14:07:16.004Z27/30 Read block 140 len 24576
157692024-02-04T14:07:16.004Z28/30 Write block 459 len 36864 data: 13 11 10 8 10 10 13 15 14
157702024-02-04T14:07:16.005Z29/30 Write block 460 len 8192 data: 12 11
157712024-02-04T14:07:16.005Z30/30 Read block 235 len 40960
157722024-02-04T14:07:16.005Z----------------------------------------------------------------
157732024-02-04T14:07:16.005Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
157742024-02-04T14:07:16.005ZStates: Active Active Active
157752024-02-04T14:07:16.005ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157762024-02-04T14:07:16.006Z 1 Acked 1000 Write 1 Done Sent Done false
157772024-02-04T14:07:16.006Z 2 Acked 1001 Read 4 Done Sent Done false
157782024-02-04T14:07:16.006Z 3 Acked 1002 Read 5 Done Sent Done false
157792024-02-04T14:07:16.006Z 4 Acked 1003 Read 5 Done Sent Done false
157802024-02-04T14:07:16.007Z 5 Acked 1004 Read 4 Done Sent Done false
157812024-02-04T14:07:16.007Z 6 Acked 1005 Read 9 Done Sent Done false
157822024-02-04T14:07:16.007Z 7 Acked 1006 Flush 0 Done Sent Done false
157832024-02-04T14:07:16.007Z 8 Acked 1007 Write 10 Done Sent Done false
157842024-02-04T14:07:16.007Z 9 Acked 1008 Write 9 Done Sent Done false
157852024-02-04T14:07:16.008Z 10 Acked 1009 Read 1 Done Sent Done false
157862024-02-04T14:07:16.008Z 11 Acked 1010 Read 1 Done Sent Done false
157872024-02-04T14:07:16.008Z 12 Acked 1011 Write 6 Done Sent Done false
157882024-02-04T14:07:16.008Z 13 Acked 1012 Read 4 Done Sent Done false
157892024-02-04T14:07:16.009Z 14 Acked 1013 Write 4 Done Sent Done false
157902024-02-04T14:07:16.009Z 15 Acked 1014 Flush 0 Done Sent Done false
157912024-02-04T14:07:16.009Z 16 Acked 1015 Write 8 Done Sent Done false
157922024-02-04T14:07:16.009Z 17 Acked 1016 Write 9 Done Sent Done false
157932024-02-04T14:07:16.010Z 18 Acked 1017 Read 4 Done Sent Done false
157942024-02-04T14:07:16.010Z 19 Acked 1018 Flush 0 Done Sent Done false
157952024-02-04T14:07:16.010Z 20 Acked 1019 Write 9 Done Sent Done false
157962024-02-04T14:07:16.010Z 21 Acked 1020 Write 4 Done Sent Done false
157972024-02-04T14:07:16.011Z 22 Acked 1021 Read 6 Done Sent Done false
157982024-02-04T14:07:16.011Z 23 Acked 1022 Write 1 Done Sent Done false
157992024-02-04T14:07:16.011Z 24 Acked 1023 Read 5 Done Sent Done false
158002024-02-04T14:07:16.011Z 25 Acked 1024 Read 8 Done Sent Done false
158012024-02-04T14:07:16.011Z 26 Acked 1025 Write 1 Done Sent Done false
158022024-02-04T14:07:16.012Z 27 Acked 1026 Read 6 Done Sent Done false
158032024-02-04T14:07:16.012Z 28 Acked 1027 Write 9 Done Sent Done false
158042024-02-04T14:07:16.012Z 29 Acked 1028 Write 2 Done Sent Done false
158052024-02-04T14:07:16.012Z 30 Acked 1029 Read 10 Sent Sent Done false
158062024-02-04T14:07:16.013Z STATES DS:0 DS:1 DS:2 TOTAL
158072024-02-04T14:07:16.013Z New 0 0 0 0
158082024-02-04T14:07:16.013Z Sent 1 30 0 31
158092024-02-04T14:07:16.013Z Done 29 0 30 59
158102024-02-04T14:07:16.014Z Skipped 0 0 0 0
158112024-02-04T14:07:16.014Z Error 0 0 0 0
158122024-02-04T14:07:16.014ZLast Flush: 1018 0 1018
158132024-02-04T14:07:16.014ZDownstairs last five completed:
158142024-02-04T14:07:16.014ZUpstairs last five completed: 30 29 28 27 26
158152024-02-04T14:07:16.015Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:14.640318484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1713}
158162024-02-04T14:07:16.015ZWrote out file "/tmp/test_repair_verify.data"
158172024-02-04T14:07:16.015Z
158182024-02-04T14:07:16.015ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158192024-02-04T14:07:16.016ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158202024-02-04T14:07:16.016Z 0 000-019 139 139 139 233 233 233 T F T <---
158212024-02-04T14:07:16.016Z 1 020-039 144 144 144 246 246 246 T F T <---
158222024-02-04T14:07:16.016Z 2 040-059 144 144 144 246 246 246 F F F
158232024-02-04T14:07:16.017Z 3 060-079 137 137 137 230 230 230 F F F
158242024-02-04T14:07:16.017Z 4 080-099 146 142 146 251 241 251 F F F <---
158252024-02-04T14:07:16.017Z 5 100-119 144 144 144 247 247 247 F F F
158262024-02-04T14:07:16.018Z 6 120-139 143 143 143 244 244 244 T F T <---
158272024-02-04T14:07:16.018Z 7 140-159 141 141 141 239 239 239 F F F
158282024-02-04T14:07:16.018Z 8 160-179 141 141 141 239 239 239 F F F
158292024-02-04T14:07:16.018Z 9 180-199 139 139 139 233 233 233 F F F
158302024-02-04T14:07:16.018Z 10 200-219 136 136 136 228 228 228 F F F
158312024-02-04T14:07:16.019Z 11 220-239 146 144 146 250 247 250 F F F <---
158322024-02-04T14:07:16.019Z 12 240-259 144 144 144 247 247 247 T F T <---
158332024-02-04T14:07:16.019Z 13 260-279 144 144 144 245 245 245 F F F
158342024-02-04T14:07:16.019Z 14 280-299 142 142 142 242 242 242 F F F
158352024-02-04T14:07:16.019Z 15 300-319 142 142 142 242 242 242 F F F
158362024-02-04T14:07:16.020Z 16 320-339 144 144 144 247 247 247 F F F
158372024-02-04T14:07:16.020Z 17 340-359 145 145 145 248 248 248 F F F
158382024-02-04T14:07:16.020Z 18 360-379 144 144 144 247 247 247 F F F
158392024-02-04T14:07:16.020Z 19 380-399 144 144 144 245 245 245 F F F
158402024-02-04T14:07:16.020Z 20 400-419 145 145 145 248 248 248 F F F
158412024-02-04T14:07:16.021Z 21 420-439 145 145 145 248 248 248 F F F
158422024-02-04T14:07:16.021Z 22 440-459 146 138 146 251 231 251 T F T <---
158432024-02-04T14:07:16.021Z 23 460-479 146 138 146 250 231 250 T F T <---
158442024-02-04T14:07:16.021Z 24 480-499 142 142 142 241 241 241 F F F
158452024-02-04T14:07:16.021Z 25 500-519 142 142 142 241 241 241 F F F
158462024-02-04T14:07:16.021Z 26 520-539 146 144 146 249 246 249 T F T <---
158472024-02-04T14:07:16.022Z 27 540-559 146 144 146 250 247 250 F F F <---
158482024-02-04T14:07:16.035Z 28 560-579 144 144 144 247 247 247 F F F
158492024-02-04T14:07:16.035Z 29 580-599 124 124 124 205 205 205 F F F
158502024-02-04T14:07:16.035ZMax gen: 146, Max flush: 251
158512024-02-04T14:07:16.036ZError: Difference in extent metadata found!
158522024-02-04T14:07:16.036ZA Difference in extent metadata is expected here
158532024-02-04T14:07:16.036ZOn loop 73
158542024-02-04T14:07:16.036Z
158552024-02-04T14:07:16.037ZVerifying data now
158562024-02-04T14:07:21.563ZNo change after verify
158572024-02-04T14:07:21.563ZLoop: 73 Downstairs dump after verify (and repair):
158582024-02-04T14:07:21.587ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158592024-02-04T14:07:21.587Z 0 000-019 147 147 147 252 252 252 F F F
158602024-02-04T14:07:21.587Z 1 020-039 147 147 147 252 252 252 F F F
158612024-02-04T14:07:21.587Z 2 040-059 144 144 144 246 246 246 F F F
158622024-02-04T14:07:21.587Z 3 060-079 137 137 137 230 230 230 F F F
158632024-02-04T14:07:21.587Z 4 080-099 146 146 146 251 251 251 F F F
158642024-02-04T14:07:21.587Z 5 100-119 144 144 144 247 247 247 F F F
158652024-02-04T14:07:21.591Z 6 120-139 147 147 147 252 252 252 F F F
158662024-02-04T14:07:21.591Z 7 140-159 141 141 141 239 239 239 F F F
158672024-02-04T14:07:21.591Z 8 160-179 141 141 141 239 239 239 F F F
158682024-02-04T14:07:21.591Z 9 180-199 139 139 139 233 233 233 F F F
158692024-02-04T14:07:21.591Z 10 200-219 136 136 136 228 228 228 F F F
158702024-02-04T14:07:21.591Z 11 220-239 146 146 146 250 250 250 F F F
158712024-02-04T14:07:21.591Z 12 240-259 147 147 147 252 252 252 F F F
158722024-02-04T14:07:21.591Z 13 260-279 144 144 144 245 245 245 F F F
158732024-02-04T14:07:21.591Z 14 280-299 142 142 142 242 242 242 F F F
158742024-02-04T14:07:21.591Z 15 300-319 142 142 142 242 242 242 F F F
158752024-02-04T14:07:21.591Z 16 320-339 144 144 144 247 247 247 F F F
158762024-02-04T14:07:21.591Z 17 340-359 145 145 145 248 248 248 F F F
158772024-02-04T14:07:21.591Z 18 360-379 144 144 144 247 247 247 F F F
158782024-02-04T14:07:21.591Z 19 380-399 144 144 144 245 245 245 F F F
158792024-02-04T14:07:21.591Z 20 400-419 145 145 145 248 248 248 F F F
158802024-02-04T14:07:21.591Z 21 420-439 145 145 145 248 248 248 F F F
158812024-02-04T14:07:21.591Z 22 440-459 147 147 147 252 252 252 F F F
158822024-02-04T14:07:21.591Z 23 460-479 147 147 147 252 252 252 F F F
158832024-02-04T14:07:21.591Z 24 480-499 142 142 142 241 241 241 F F F
158842024-02-04T14:07:21.591Z 25 500-519 142 142 142 241 241 241 F F F
158852024-02-04T14:07:21.592Z 26 520-539 147 147 147 252 252 252 F F F
158862024-02-04T14:07:21.592Z 27 540-559 146 146 146 250 250 250 F F F
158872024-02-04T14:07:21.592Z 28 560-579 144 144 144 247 247 247 F F F
158882024-02-04T14:07:21.592Z 29 580-599 124 124 124 205 205 205 F F F
158892024-02-04T14:07:21.592ZMax gen: 147, Max flush: 252
158902024-02-04T14:07:21.592Z
158912024-02-04T14:07:21.592ZBegin loop 74
158922024-02-04T14:07:21.592ZDownstairs to restart: 2
158932024-02-04T14:07:21.622Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.65398822Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721}
158942024-02-04T14:07:21.622Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.654466119Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721}
158952024-02-04T14:07:21.622Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.654559614Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721}
158962024-02-04T14:07:21.626Z{"msg":"Crucible 528a1e17-e9dd-4834-837f-07633ac72e63 has session id: aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.654601258Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
158972024-02-04T14:07:21.626Z{"msg":"Upstairs opts: Upstairs UUID: 528a1e17-e9dd-4834-837f-07633ac72e63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
158982024-02-04T14:07:21.626Z,"time":"2024-02-04T14:07:20.654681851Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
158992024-02-04T14:07:21.626Z{"msg":"Crucible stats registered with UUID: 528a1e17-e9dd-4834-837f-07633ac72e63","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.654728754Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159002024-02-04T14:07:21.626Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.654793987Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721}
159012024-02-04T14:07:21.626Z{"msg":"528a1e17-e9dd-4834-837f-07633ac72e63 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655031959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159022024-02-04T14:07:21.626Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655087849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159032024-02-04T14:07:21.626Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655114169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159042024-02-04T14:07:21.627Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655137162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159052024-02-04T14:07:21.627Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655160283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159062024-02-04T14:07:21.627Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655182796Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159072024-02-04T14:07:21.627Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655206485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159082024-02-04T14:07:21.627Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.65527089Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"io task","client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159092024-02-04T14:07:21.627Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655300687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"io task","client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159102024-02-04T14:07:21.627Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655344014Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"io task","client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159112024-02-04T14:07:21.627Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655758485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"io task","client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159122024-02-04T14:07:21.627Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655842476Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"io task","client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159132024-02-04T14:07:21.627Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.655889214Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"io task","client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159142024-02-04T14:07:21.627Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.657046014Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159152024-02-04T14:07:21.627Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.657111464Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159162024-02-04T14:07:21.627Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.657308775Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159172024-02-04T14:07:21.627Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.657374502Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159182024-02-04T14:07:21.627Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.657876862Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159192024-02-04T14:07:21.627Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.658141281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159202024-02-04T14:07:21.632Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.65878249Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159212024-02-04T14:07:21.632Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.658894777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159222024-02-04T14:07:21.632Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.659111683Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159232024-02-04T14:07:21.632Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.659191451Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159242024-02-04T14:07:21.632Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.659336289Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159252024-02-04T14:07:21.632Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.659401261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159262024-02-04T14:07:21.632Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.65944621Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159272024-02-04T14:07:21.632Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.659473504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159282024-02-04T14:07:21.633Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.659857331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159292024-02-04T14:07:21.633Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.660571747Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159302024-02-04T14:07:21.633Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.660652524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159312024-02-04T14:07:21.633Z{"msg":"[0]R flush_numbers[0..12]: [252, 252, 246, 230, 251, 247, 252, 239, 239, 233, 228, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.660730965Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159322024-02-04T14:07:21.633Z{"msg":"[0]R generation[0..12]: [147, 147, 144, 137, 146, 144, 147, 141, 141, 139, 136, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.660782262Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159332024-02-04T14:07:21.633Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
159342024-02-04T14:07:21.633Z2024-02-04T14:07:20.660808447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159352024-02-04T14:07:21.633Z{"msg":"[1]R flush_numbers[0..12]: [252, 252, 246, 230, 251, 247, 252, 239, 239, 233, 228, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.660893639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159362024-02-04T14:07:21.633Z{"msg":"[1]R generation[0..12]: [147, 147, 144, 137, 146, 144, 147, 141, 141, 139, 136, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.660941174Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159372024-02-04T14:07:21.633Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.660967522Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159382024-02-04T14:07:21.633Z{"msg":"[2]R flush_numbers[0..12]: [252, 252, 246, 230, 251, 247, 252, 239, 239, 233, 228, 250]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661011161Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159392024-02-04T14:07:21.633Z{"msg":"[2]R generation[0..12]: [147, 147, 144, 137, 146, 144, 147, 141, 141, 139, 136, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661036014Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159402024-02-04T14:07:21.633Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.66108371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159412024-02-04T14:07:21.633Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661107933Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159422024-02-04T14:07:21.633Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661176244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159432024-02-04T14:07:21.633Z{"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661209362Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159442024-02-04T14:07:21.633Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661232891Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"":"downstairs","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159452024-02-04T14:07:21.633Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.66125508Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159462024-02-04T14:07:21.633Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661275687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"0","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159472024-02-04T14:07:21.633Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661305565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"1","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159482024-02-04T14:07:21.633Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661329458Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"client":"2","":"downstairs","session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159492024-02-04T14:07:21.633Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661376557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159502024-02-04T14:07:21.633Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661399034Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159512024-02-04T14:07:21.633Z{"msg":"528a1e17-e9dd-4834-837f-07633ac72e63 is now active with session: aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661454713Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159522024-02-04T14:07:21.634Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661476061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721,"session_id":"aa1be231-acfb-4a19-b1a4-5fa74d0b3f6c"}
159532024-02-04T14:07:21.634Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:20.661496596Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721}
159542024-02-04T14:07:21.634ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159552024-02-04T14:07:21.634ZLoading write count information from file "/tmp/test_repair_verify.data"
159562024-02-04T14:07:21.634ZRun Repair workload
159572024-02-04T14:07:21.634Z01/30 Write block 245 len 20480 data: 11 13 14 18 17
159582024-02-04T14:07:21.637Z02/30 Read block 488 len 8192
159592024-02-04T14:07:21.637Z03/30 Read block 366 len 24576
159602024-02-04T14:07:21.637Z04/30 Write block 203 len 36864 data: 15 17 14 13 12 10 8 7 9
159612024-02-04T14:07:21.637Z05/30 Read block 132 len 4096
159622024-02-04T14:07:21.640Z06/30 Flush
159632024-02-04T14:07:21.643Z07/30 Write block 454 len 20480 data: 14 15 14 14 13
159642024-02-04T14:07:21.644Z08/30 Read block 531 len 20480
159652024-02-04T14:07:21.646Z09/30 Write block 345 len 36864 data: 12 11 12 14 17 12 14 14 14
159662024-02-04T14:07:21.647Z10/30 Write block 236 len 16384 data: 12 16 13 12
159672024-02-04T14:07:21.647Z11/30 Flush
159682024-02-04T14:07:21.650Z12/30 Write block 124 len 4096 data: 11
159692024-02-04T14:07:21.650Z13/30 Write block 132 len 12288 data: 9 6 8
159702024-02-04T14:07:21.653Z14/30 Read block 254 len 28672
159712024-02-04T14:07:21.653Z15/30 Flush
159722024-02-04T14:07:21.656Z16/30 Write block 437 len 40960 data: 11 9 7 10 10 11 13 14 15 16
159732024-02-04T14:07:21.656Z17/30 Write block 540 len 24576 data: 11 11 10 13 11 10
159742024-02-04T14:07:21.659Z18/30 Write block 321 len 36864 data: 9 9 8 7 7 8 7 6 6
159752024-02-04T14:07:21.659Z19/30 Write block 278 len 8192 data: 8 8
159762024-02-04T14:07:21.659Z20/30 Read block 592 len 28672
159772024-02-04T14:07:21.662Z21/30 Read block 550 len 24576
159782024-02-04T14:07:21.665Z22/30 Read block 144 len 28672
159792024-02-04T14:07:21.665Z23/30 Flush
159802024-02-04T14:07:21.670Z24/30 Write block 401 len 16384 data: 11 13 12 13
159812024-02-04T14:07:21.671Z25/30 Flush
159822024-02-04T14:07:21.674Z26/30 Write block 574 len 20480 data: 9 10 13 13 13
159832024-02-04T14:07:21.674Z27/30 Write block 272 len 28672 data: 6 8 7 7 6 6 9
159842024-02-04T14:07:21.674Z28/30 Write block 427 len 40960 data: 8 8 7 10 10 10 10 14 14 13
159852024-02-04T14:07:21.676Z29/30 Read block 251 len 20480
159862024-02-04T14:07:21.680Z30/30 Write block 344 len 40960 data: 10 13 12 13 15 18 13 15 15 15
159872024-02-04T14:07:21.680Z----------------------------------------------------------------
159882024-02-04T14:07:21.680Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
159892024-02-04T14:07:21.680ZStates: Active Active Active
159902024-02-04T14:07:21.680ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159912024-02-04T14:07:21.680Z 1 Acked 1000 Write 5 Done Done Done false
159922024-02-04T14:07:21.680Z 2 Acked 1001 Read 2 Done Done Done false
159932024-02-04T14:07:21.680Z 3 Acked 1002 Read 6 Done Done Sent false
159942024-02-04T14:07:21.680Z 4 Acked 1003 Write 9 Done Done Sent false
159952024-02-04T14:07:21.680Z 5 Acked 1004 Read 1 Done Done Sent false
159962024-02-04T14:07:21.680Z 6 Acked 1005 Flush 0 Done Done Sent false
159972024-02-04T14:07:21.680Z 7 Acked 1006 Write 5 Done Done Sent false
159982024-02-04T14:07:21.680Z 8 Acked 1007 Read 5 Done Done Sent false
159992024-02-04T14:07:21.680Z 9 Acked 1008 Write 9 Done Done Sent false
160002024-02-04T14:07:21.680Z 10 Acked 1009 Write 4 Done Done Sent false
160012024-02-04T14:07:21.681Z 11 Acked 1010 Flush 0 Done Done Sent false
160022024-02-04T14:07:21.681Z 12 Acked 1011 Write 1 Done Done Sent false
160032024-02-04T14:07:21.681Z 13 Acked 1012 Write 3 Done Done Sent false
160042024-02-04T14:07:21.681Z 14 Acked 1013 Read 7 Done Done Sent false
160052024-02-04T14:07:21.681Z 15 Acked 1014 Flush 0 Done Done Sent false
160062024-02-04T14:07:21.681Z 16 Acked 1015 Write 10 Done Done Sent false
160072024-02-04T14:07:21.681Z 17 Acked 1016 Write 6 Done Done Sent false
160082024-02-04T14:07:21.681Z 18 Acked 1017 Write 9 Done Done Sent false
160092024-02-04T14:07:21.681Z 19 Acked 1018 Write 2 Done Done Sent false
160102024-02-04T14:07:21.681Z 20 Acked 1019 Read 7 Done Done Sent false
160112024-02-04T14:07:21.681Z 21 Acked 1020 Read 6 Done Done Sent false
160122024-02-04T14:07:21.681Z 22 Acked 1021 Read 7 Done Done Sent false
160132024-02-04T14:07:21.681Z 23 Acked 1022 Flush 0 Done Done Sent false
160142024-02-04T14:07:21.681Z 24 Acked 1023 Write 4 Done Done Sent false
160152024-02-04T14:07:21.681Z 25 Acked 1024 Flush 0 Done Done Sent false
160162024-02-04T14:07:21.681Z 26 Acked 1025 Write 5 Done Done Sent false
160172024-02-04T14:07:21.681Z 27 Acked 1026 Write 7 Done Done Sent false
160182024-02-04T14:07:21.681Z 28 Acked 1027 Write 10 Done Done Sent false
160192024-02-04T14:07:21.681Z 29 Acked 1028 Read 5 Done Done Sent false
160202024-02-04T14:07:21.681Z 30 Acked 1029 Write 10 Sent Sent Sent false
160212024-02-04T14:07:21.681Z STATES DS:0 DS:1 DS:2 TOTAL
160222024-02-04T14:07:21.681Z New 0 0 0 0
160232024-02-04T14:07:21.681Z Sent 1 1 28 30
160242024-02-04T14:07:21.681Z Done 29 29 2 60
160252024-02-04T14:07:21.681Z Skipped 0 0 0 0
160262024-02-04T14:07:21.682Z Error 0 0 0 0
160272024-02-04T14:07:21.682ZLast Flush: 1024 1024 0
160282024-02-04T14:07:21.682ZDownstairs last five completed:
160292024-02-04T14:07:21.682ZUpstairs last five completed: 30 29 28 27 26
160302024-02-04T14:07:21.682Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:20.711370356Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1721}
160312024-02-04T14:07:21.682ZWrote out file "/tmp/test_repair_verify.data"
160322024-02-04T14:07:21.686Z
160332024-02-04T14:07:21.692ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160342024-02-04T14:07:21.717ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160352024-02-04T14:07:21.717Z 0 000-019 147 147 147 252 252 252 F F F
160362024-02-04T14:07:21.717Z 1 020-039 147 147 147 252 252 252 F F F
160372024-02-04T14:07:21.717Z 2 040-059 144 144 144 246 246 246 F F F
160382024-02-04T14:07:21.717Z 3 060-079 137 137 137 230 230 230 F F F
160392024-02-04T14:07:21.717Z 4 080-099 146 146 146 251 251 251 F F F
160402024-02-04T14:07:21.717Z 5 100-119 144 144 144 247 247 247 F F F
160412024-02-04T14:07:21.721Z 6 120-139 148 148 147 255 255 252 F F F <---
160422024-02-04T14:07:21.721Z 7 140-159 141 141 141 239 239 239 F F F
160432024-02-04T14:07:21.721Z 8 160-179 141 141 141 239 239 239 F F F
160442024-02-04T14:07:21.721Z 9 180-199 139 139 139 233 233 233 F F F
160452024-02-04T14:07:21.721Z 10 200-219 148 148 136 253 253 228 F F F <---
160462024-02-04T14:07:21.721Z 11 220-239 148 148 146 254 254 250 F F F <---
160472024-02-04T14:07:21.721Z 12 240-259 148 148 147 253 253 252 F F T <---
160482024-02-04T14:07:21.721Z 13 260-279 148 148 144 256 256 245 T T F <---
160492024-02-04T14:07:21.721Z 14 280-299 142 142 142 242 242 242 F F F
160502024-02-04T14:07:21.721Z 15 300-319 142 142 142 242 242 242 F F F
160512024-02-04T14:07:21.721Z 16 320-339 148 148 144 256 256 247 F F F <---
160522024-02-04T14:07:21.721Z 17 340-359 148 148 145 254 254 248 T T F <---
160532024-02-04T14:07:21.721Z 18 360-379 144 144 144 247 247 247 F F F
160542024-02-04T14:07:21.721Z 19 380-399 144 144 144 245 245 245 F F F
160552024-02-04T14:07:21.721Z 20 400-419 148 148 145 257 257 248 F F F <---
160562024-02-04T14:07:21.721Z 21 420-439 148 148 145 256 256 248 T T F <---
160572024-02-04T14:07:21.721Z 22 440-459 148 148 147 256 256 252 F F F <---
160582024-02-04T14:07:21.721Z 23 460-479 147 147 147 252 252 252 F F F
160592024-02-04T14:07:21.721Z 24 480-499 142 142 142 241 241 241 F F F
160602024-02-04T14:07:21.721Z 25 500-519 142 142 142 241 241 241 F F F
160612024-02-04T14:07:21.722Z 26 520-539 147 147 147 252 252 252 F F F
160622024-02-04T14:07:21.722Z 27 540-559 148 148 146 256 256 250 F F F <---
160632024-02-04T14:07:21.722Z 28 560-579 144 144 144 247 247 247 T T F <---
160642024-02-04T14:07:21.722Z 29 580-599 124 124 124 205 205 205 F F F
160652024-02-04T14:07:21.722ZMax gen: 148, Max flush: 257
160662024-02-04T14:07:21.722ZError: Difference in extent metadata found!
160672024-02-04T14:07:21.725ZA Difference in extent metadata is expected here
160682024-02-04T14:07:21.725ZOn loop 74
160692024-02-04T14:07:21.725Z
160702024-02-04T14:07:21.729ZVerifying data now
160712024-02-04T14:07:22.774ZNo change after verify
160722024-02-04T14:07:22.774ZLoop: 74 Downstairs dump after verify (and repair):
160732024-02-04T14:07:22.795ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160742024-02-04T14:07:22.795Z 0 000-019 147 147 147 252 252 252 F F F
160752024-02-04T14:07:22.796Z 1 020-039 147 147 147 252 252 252 F F F
160762024-02-04T14:07:22.796Z 2 040-059 144 144 144 246 246 246 F F F
160772024-02-04T14:07:22.803Z 3 060-079 137 137 137 230 230 230 F F F
160782024-02-04T14:07:22.803Z 4 080-099 146 146 146 251 251 251 F F F
160792024-02-04T14:07:22.803Z 5 100-119 144 144 144 247 247 247 F F F
160802024-02-04T14:07:22.803Z 6 120-139 148 148 148 255 255 255 F F F
160812024-02-04T14:07:22.803Z 7 140-159 141 141 141 239 239 239 F F F
160822024-02-04T14:07:22.803Z 8 160-179 141 141 141 239 239 239 F F F
160832024-02-04T14:07:22.803Z 9 180-199 139 139 139 233 233 233 F F F
160842024-02-04T14:07:22.803Z 10 200-219 148 148 148 253 253 253 F F F
160852024-02-04T14:07:22.803Z 11 220-239 148 148 148 254 254 254 F F F
160862024-02-04T14:07:22.803Z 12 240-259 148 148 148 253 253 253 F F F
160872024-02-04T14:07:22.803Z 13 260-279 149 149 149 258 258 258 F F F
160882024-02-04T14:07:22.804Z 14 280-299 142 142 142 242 242 242 F F F
160892024-02-04T14:07:22.804Z 15 300-319 142 142 142 242 242 242 F F F
160902024-02-04T14:07:22.804Z 16 320-339 148 148 148 256 256 256 F F F
160912024-02-04T14:07:22.804Z 17 340-359 149 149 149 258 258 258 F F F
160922024-02-04T14:07:22.804Z 18 360-379 144 144 144 247 247 247 F F F
160932024-02-04T14:07:22.804Z 19 380-399 144 144 144 245 245 245 F F F
160942024-02-04T14:07:22.804Z 20 400-419 148 148 148 257 257 257 F F F
160952024-02-04T14:07:22.804Z 21 420-439 149 149 149 258 258 258 F F F
160962024-02-04T14:07:22.804Z 22 440-459 148 148 148 256 256 256 F F F
160972024-02-04T14:07:22.804Z 23 460-479 147 147 147 252 252 252 F F F
160982024-02-04T14:07:22.804Z 24 480-499 142 142 142 241 241 241 F F F
160992024-02-04T14:07:22.804Z 25 500-519 142 142 142 241 241 241 F F F
161002024-02-04T14:07:22.804Z 26 520-539 147 147 147 252 252 252 F F F
161012024-02-04T14:07:22.804Z 27 540-559 148 148 148 256 256 256 F F F
161022024-02-04T14:07:22.804Z 28 560-579 149 149 149 258 258 258 F F F
161032024-02-04T14:07:22.804Z 29 580-599 124 124 124 205 205 205 F F F
161042024-02-04T14:07:22.804ZMax gen: 149, Max flush: 258
161052024-02-04T14:07:22.804Z
161062024-02-04T14:07:22.804ZBegin loop 75
161072024-02-04T14:07:22.804ZDownstairs to restart: 2
161082024-02-04T14:07:22.827Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.859079181Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729}
161092024-02-04T14:07:22.827Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.859546827Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729}
161102024-02-04T14:07:22.839Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.859658432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729}
161112024-02-04T14:07:22.839Z{"msg":"Crucible a631babb-f7b7-4e27-8ad7-aa3c6a750930 has session id: 647c23e2-79fa-4922-ab9a-b603d311f6df","v":0,"name":"crucible","level"Crucible runtime is spawned
161122024-02-04T14:07:22.839Z:30,"time":"2024-02-04T14:07:21.85974449Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161132024-02-04T14:07:22.839Z{"msg":"Upstairs opts: Upstairs UUID: a631babb-f7b7-4e27-8ad7-aa3c6a750930, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.859846173Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161142024-02-04T14:07:22.839Z{"msg":"Crucible stats registered with UUID: a631babb-f7b7-4e27-8ad7-aa3c6a750930","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.859902192Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161152024-02-04T14:07:22.839Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.859926287Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729}
161162024-02-04T14:07:22.840Z{"msg":"a631babb-f7b7-4e27-8ad7-aa3c6a750930 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.859965334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161172024-02-04T14:07:22.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.859987216Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161182024-02-04T14:07:22.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.860010769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161192024-02-04T14:07:22.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.860034003Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161202024-02-04T14:07:22.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.860196494Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161212024-02-04T14:07:22.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.860277333Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161222024-02-04T14:07:22.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.86032157Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161232024-02-04T14:07:22.840Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.860360359Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"io task","client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161242024-02-04T14:07:22.840Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.860400948Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"io task","client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161252024-02-04T14:07:22.840Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.860853589Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"io task","client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161262024-02-04T14:07:22.840Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.860991278Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"io task","client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161272024-02-04T14:07:22.840Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.861131618Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"io task","client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161282024-02-04T14:07:22.841Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.861207116Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"io task","client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161292024-02-04T14:07:22.841Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.862366253Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161302024-02-04T14:07:22.841Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.862499549Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161312024-02-04T14:07:22.841Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.862574122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161322024-02-04T14:07:22.841Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.862616665Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161332024-02-04T14:07:22.841Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.862753117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161342024-02-04T14:07:22.841Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.86278395Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161352024-02-04T14:07:22.841Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.863437633Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161362024-02-04T14:07:22.841Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.863505515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161372024-02-04T14:07:22.841Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.863700145Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161382024-02-04T14:07:22.841Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.864496443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161392024-02-04T14:07:22.841Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.86458659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161402024-02-04T14:07:22.841Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.864687386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161412024-02-04T14:07:22.842Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.864740557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161422024-02-04T14:07:22.842Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.864769007Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161432024-02-04T14:07:22.842Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.864824959Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161442024-02-04T14:07:22.842Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.864853509Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161452024-02-04T14:07:22.842Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.864908198Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161462024-02-04T14:07:22.842Z{"msg":"[0]R flush_numbers[0..12]: [252, 252, 246, 230, 251, 247, 255, 239, 239, 233, 253, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.864940194Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161472024-02-04T14:07:22.842Z{"msg":"[0]R generation[0..12]: [147, 147, 144, 137, 146, 144, 148, 141, 141, 139, 148, 148]","vWait for a query_work_queue command to finish before sending IO
161482024-02-04T14:07:22.842Z":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865012643Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161492024-02-04T14:07:22.873Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865075757Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161502024-02-04T14:07:22.873Z{"msg":"[1]R flush_numbers[0..12]: [252, 252, 246, 230, 251, 247, 255, 239, 239, 233, 253, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865181905Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161512024-02-04T14:07:22.873Z{"msg":"[1]R generation[0..12]: [147, 147, 144, 137, 146, 144, 148, 141, 141, 139, 148, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865214226Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161522024-02-04T14:07:22.873Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865286753Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161532024-02-04T14:07:22.873Z{"msg":"[2]R flush_numbers[0..12]: [252, 252, 246, 230, 251, 247, 255, 239, 239, 233, 253, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.86534904Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161542024-02-04T14:07:22.873Z{"msg":"[2]R generation[0..12]: [147, 147, 144, 137, 146, 144, 148, 141, 141, 139, 148, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865383085Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161552024-02-04T14:07:22.874Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.86545078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161562024-02-04T14:07:22.874Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865482556Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161572024-02-04T14:07:22.874Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865549344Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161582024-02-04T14:07:22.874Z{"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865577963Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161592024-02-04T14:07:22.874Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865644086Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"":"downstairs","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161602024-02-04T14:07:22.874Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865678341Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161612024-02-04T14:07:22.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.86574332Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"0","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161622024-02-04T14:07:22.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865783516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161632024-02-04T14:07:22.874Z:1729,"client":"1","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161642024-02-04T14:07:22.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865890729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"client":"2","":"downstairs","session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161652024-02-04T14:07:22.874Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865945844Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161662024-02-04T14:07:22.874Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.865973075Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161672024-02-04T14:07:22.874Z{"msg":"a631babb-f7b7-4e27-8ad7-aa3c6a750930 is now active with session: 647c23e2-79fa-4922-ab9a-b603d311f6df","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.866041354Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161682024-02-04T14:07:22.874Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.866065993Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729,"session_id":"647c23e2-79fa-4922-ab9a-b603d311f6df"}
161692024-02-04T14:07:22.874Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:21.866120889Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729}
161702024-02-04T14:07:22.875ZLoading write count information from file "/tmp/test_repair_verify.data"
161712024-02-04T14:07:22.875ZRun Repair workload
161722024-02-04T14:07:22.875Z01/30 Write block 62 len 8192 data: 12 11
161732024-02-04T14:07:22.875Z02/30 Write block 523 len 40960 data: 11 14 13 16 15 16 13 11 8 8
161742024-02-04T14:07:22.875Z03/30 Write block 319 len 12288 data: 11 10 10
161752024-02-04T14:07:22.875Z04/30 Read block 459 len 16384
161762024-02-04T14:07:22.875Z05/30 Read block 510 len 16384
161772024-02-04T14:07:22.875Z06/30 Read block 24 len 8192
161782024-02-04T14:07:22.875Z07/30 Write block 458 len 20480 data: 14 14 13 12 9
161792024-02-04T14:07:22.875Z08/30 Read block 446 len 24576
161802024-02-04T14:07:22.875Z09/30 Flush
161812024-02-04T14:07:22.879Z10/30 Write block 274 len 16384 data: 8 8 7 7
161822024-02-04T14:07:22.880Z11/30 Write block 564 len 28672 data: 16 17 15 13 12 10 12
161832024-02-04T14:07:22.880Z12/30 Read block 410 len 28672
161842024-02-04T14:07:22.880Z13/30 Write block 210 len 20480 data: 8 10 10 9 11
161852024-02-04T14:07:22.880Z14/30 Read block 521 len 36864
161862024-02-04T14:07:22.880Z15/30 Write block 564 len 36864 data: 17 18 16 14 13 11 13 11 10
161872024-02-04T14:07:22.880Z16/30 Read block 28 len 32768
161882024-02-04T14:07:22.880Z17/30 Read block 528 len 16384
161892024-02-04T14:07:22.880Z18/30 Read block 196 len 12288
161902024-02-04T14:07:22.880Z19/30 Write block 57 len 28672 data: 11 9 11 13 11 13 12
161912024-02-04T14:07:22.880Z20/30 Read block 114 len 8192
161922024-02-04T14:07:22.880Z21/30 Write block 304 len 32768 data: 10 12 11 11 10 10 12 11
161932024-02-04T14:07:22.880Z22/30 Write block 125 len 8192 data: 13 13
161942024-02-04T14:07:22.880Z23/30 Write block 194 len 8192 data: 9 8
161952024-02-04T14:07:22.881Z24/30 Read block 263 len 20480
161962024-02-04T14:07:22.881Z25/30 Read block 492 len 24576
161972024-02-04T14:07:22.881Z26/30 Flush
161982024-02-04T14:07:22.881Z27/30 Write block 295 len 20480 data: 7 6 7 9 10
161992024-02-04T14:07:22.881Z28/30 Read block 257 len 40960
162002024-02-04T14:07:22.881Z29/30 Write block 253 len 12288 data: 10 12 13
162012024-02-04T14:07:22.881Z30/30 Read block 421 len 20480
162022024-02-04T14:07:22.885Z----------------------------------------------------------------
162032024-02-04T14:07:22.885Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
162042024-02-04T14:07:22.885ZStates: Active Active Active
162052024-02-04T14:07:22.885ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162062024-02-04T14:07:22.885Z 1 Acked 1000 Write 2 Done Done Done false
162072024-02-04T14:07:22.885Z 2 Acked 1001 Write 10 Done Done Done false
162082024-02-04T14:07:22.885Z 3 Acked 1002 Write 3 Done Done Sent false
162092024-02-04T14:07:22.885Z 4 Acked 1003 Read 4 Done Done Sent false
162102024-02-04T14:07:22.885Z 5 Acked 1004 Read 4 Done Done Sent false
162112024-02-04T14:07:22.885Z 6 Acked 1005 Read 2 Done Done Sent false
162122024-02-04T14:07:22.886Z 7 Acked 1006 Write 5 Done Done Sent false
162132024-02-04T14:07:22.886Z 8 Acked 1007 Read 6 Done Done Sent false
162142024-02-04T14:07:22.886Z 9 Acked 1008 Flush 0 Done Done Sent false
162152024-02-04T14:07:22.886Z 10 Acked 1009 Write 4 Done Done Sent false
162162024-02-04T14:07:22.886Z 11 Acked 1010 Write 7 Done Done Sent false
162172024-02-04T14:07:22.886Z 12 Acked 1011 Read 7 Done Done Sent false
162182024-02-04T14:07:22.886Z 13 Acked 1012 Write 5 Done Done Sent false
162192024-02-04T14:07:22.886Z 14 Acked 1013 Read 9 Done Done Sent false
162202024-02-04T14:07:22.886Z 15 Acked 1014 Write 9 Done Done Sent false
162212024-02-04T14:07:22.886Z 16 Acked 1015 Read 8 Done Done Sent false
162222024-02-04T14:07:22.886Z 17 Acked 1016 Read 4 Done Done Sent false
162232024-02-04T14:07:22.886Z 18 Acked 1017 Read 3 Done Done Sent false
162242024-02-04T14:07:22.887Z 19 Acked 1018 Write 7 Done Done Sent false
162252024-02-04T14:07:22.887Z 20 Acked 1019 Read 2 Done Done Sent false
162262024-02-04T14:07:22.887Z 21 Acked 1020 Write 8 Done Done Sent false
162272024-02-04T14:07:22.887Z 22 Acked 1021 Write 2 Done Done Sent false
162282024-02-04T14:07:22.887Z 23 Acked 1022 Write 2 Done Done Sent false
162292024-02-04T14:07:22.887Z 24 Acked 1023 Read 5 Done Done Sent false
162302024-02-04T14:07:22.887Z 25 Acked 1024 Read 6 Done Done Sent false
162312024-02-04T14:07:22.887Z 26 Acked 1025 Flush 0 Done Done Sent false
162322024-02-04T14:07:22.887Z 27 Acked 1026 Write 5 Done Done Sent false
162332024-02-04T14:07:22.887Z 28 Acked 1027 Read 10 Done Done Sent false
162342024-02-04T14:07:22.887Z 29 Acked 1028 Write 3 Done Done Sent false
162352024-02-04T14:07:22.887Z 30 Acked 1029 Read 5 Sent Done Sent false
162362024-02-04T14:07:22.888Z STATES DS:0 DS:1 DS:2 TOTAL
162372024-02-04T14:07:22.888Z New 0 0 0 0
162382024-02-04T14:07:22.888Z Sent 1 0 28 29
162392024-02-04T14:07:22.888Z Done 29 30 2 61
162402024-02-04T14:07:22.888Z Skipped 0 0 0 0
162412024-02-04T14:07:22.888Z Error 0 0 0 0
162422024-02-04T14:07:22.888ZLast Flush: 1025 1025 0
162432024-02-04T14:07:22.888ZDownstairs last five completed:
162442024-02-04T14:07:22.888ZUpstairs last five completed: 30 29 28 27 26
162452024-02-04T14:07:22.888Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:21.913186567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1729}
162462024-02-04T14:07:22.888ZWrote out file "/tmp/test_repair_verify.data"
162472024-02-04T14:07:22.892Z
162482024-02-04T14:07:22.895ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162492024-02-04T14:07:22.921ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162502024-02-04T14:07:22.921Z 0 000-019 147 147 147 252 252 252 F F F
162512024-02-04T14:07:22.921Z 1 020-039 147 147 147 252 252 252 F F F
162522024-02-04T14:07:22.941Z 2 040-059 150 150 144 260 260 246 F F F <---
162532024-02-04T14:07:22.941Z 3 060-079 150 150 137 260 260 230 F F T <---
162542024-02-04T14:07:22.941Z 4 080-099 146 146 146 251 251 251 F F F
162552024-02-04T14:07:22.942Z 5 100-119 144 144 144 247 247 247 F F F
162562024-02-04T14:07:22.942Z 6 120-139 150 150 148 260 260 255 F F F <---
162572024-02-04T14:07:22.942Z 7 140-159 141 141 141 239 239 239 F F F
162582024-02-04T14:07:22.942Z 8 160-179 141 141 141 239 239 239 F F F
162592024-02-04T14:07:22.942Z 9 180-199 150 150 139 260 260 233 F F F <---
162602024-02-04T14:07:22.942Z 10 200-219 150 150 148 260 260 253 F F F <---
162612024-02-04T14:07:22.942Z 11 220-239 148 148 148 254 254 254 F F F
162622024-02-04T14:07:22.942Z 12 240-259 148 148 148 253 253 253 T T F <---
162632024-02-04T14:07:22.942Z 13 260-279 150 150 149 260 260 258 F F F <---
162642024-02-04T14:07:22.942Z 14 280-299 142 142 142 242 242 242 T T F <---
162652024-02-04T14:07:22.942Z 15 300-319 150 150 142 260 260 242 F F F <---
162662024-02-04T14:07:22.942Z 16 320-339 150 150 148 259 259 256 F F F <---
162672024-02-04T14:07:22.942Z 17 340-359 149 149 149 258 258 258 F F F
162682024-02-04T14:07:22.942Z 18 360-379 144 144 144 247 247 247 F F F
162692024-02-04T14:07:22.943Z 19 380-399 144 144 144 245 245 245 F F F
162702024-02-04T14:07:22.943Z 20 400-419 148 148 148 257 257 257 F F F
162712024-02-04T14:07:22.943Z 21 420-439 149 149 149 258 258 258 F F F
162722024-02-04T14:07:22.943Z 22 440-459 150 150 148 259 259 256 F F F <---
162732024-02-04T14:07:22.943Z 23 460-479 150 150 147 259 259 252 F F F <---
162742024-02-04T14:07:22.943Z 24 480-499 142 142 142 241 241 241 F F F
162752024-02-04T14:07:22.943Z 25 500-519 142 142 142 241 241 241 F F F
162762024-02-04T14:07:22.943Z 26 520-539 150 150 147 259 259 252 F F T <---
162772024-02-04T14:07:22.943Z 27 540-559 148 148 148 256 256 256 F F F
162782024-02-04T14:07:22.943Z 28 560-579 150 150 149 260 260 258 F F F <---
162792024-02-04T14:07:22.943Z 29 580-599 124 124 124 205 205 205 F F F
162802024-02-04T14:07:22.943ZMax gen: 150, Max flush: 260
162812024-02-04T14:07:22.943ZError: Difference in extent metadata found!
162822024-02-04T14:07:22.943ZA Difference in extent metadata is expected here
162832024-02-04T14:07:22.943ZOn loop 75
162842024-02-04T14:07:22.943Z
162852024-02-04T14:07:22.943ZVerifying data now
162862024-02-04T14:07:29.117ZNo change after verify
162872024-02-04T14:07:29.117ZLoop: 75 Downstairs dump after verify (and repair):
162882024-02-04T14:07:29.142ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162892024-02-04T14:07:29.142Z 0 000-019 147 147 147 252 252 252 F F F
162902024-02-04T14:07:29.142Z 1 020-039 147 147 147 252 252 252 F F F
162912024-02-04T14:07:29.142Z 2 040-059 150 150 150 260 260 260 F F F
162922024-02-04T14:07:29.143Z 3 060-079 150 150 150 260 260 260 F F F
162932024-02-04T14:07:29.153Z 4 080-099 146 146 146 251 251 251 F F F
162942024-02-04T14:07:29.153Z 5 100-119 144 144 144 247 247 247 F F F
162952024-02-04T14:07:29.153Z 6 120-139 150 150 150 260 260 260 F F F
162962024-02-04T14:07:29.154Z 7 140-159 141 141 141 239 239 239 F F F
162972024-02-04T14:07:29.154Z 8 160-179 141 141 141 239 239 239 F F F
162982024-02-04T14:07:29.154Z 9 180-199 150 150 150 260 260 260 F F F
162992024-02-04T14:07:29.167Z 10 200-219 150 150 150 260 260 260 F F F
163002024-02-04T14:07:29.167Z 11 220-239 148 148 148 254 254 254 F F F
163012024-02-04T14:07:29.167Z 12 240-259 151 151 151 261 261 261 F F F
163022024-02-04T14:07:29.167Z 13 260-279 150 150 150 260 260 260 F F F
163032024-02-04T14:07:29.167Z 14 280-299 151 151 151 261 261 261 F F F
163042024-02-04T14:07:29.167Z 15 300-319 150 150 150 260 260 260 F F F
163052024-02-04T14:07:29.167Z 16 320-339 150 150 150 259 259 259 F F F
163062024-02-04T14:07:29.168Z 17 340-359 149 149 149 258 258 258 F F F
163072024-02-04T14:07:29.168Z 18 360-379 144 144 144 247 247 247 F F F
163082024-02-04T14:07:29.168Z 19 380-399 144 144 144 245 245 245 F F F
163092024-02-04T14:07:29.168Z 20 400-419 148 148 148 257 257 257 F F F
163102024-02-04T14:07:29.168Z 21 420-439 149 149 149 258 258 258 F F F
163112024-02-04T14:07:29.168Z 22 440-459 150 150 150 259 259 259 F F F
163122024-02-04T14:07:29.168Z 23 460-479 150 150 150 259 259 259 F F F
163132024-02-04T14:07:29.169Z 24 480-499 142 142 142 241 241 241 F F F
163142024-02-04T14:07:29.169Z 25 500-519 142 142 142 241 241 241 F F F
163152024-02-04T14:07:29.169Z 26 520-539 150 150 150 259 259 259 F F F
163162024-02-04T14:07:29.169Z 27 540-559 148 148 148 256 256 256 F F F
163172024-02-04T14:07:29.169Z 28 560-579 150 150 150 260 260 260 F F F
163182024-02-04T14:07:29.169Z 29 580-599 124 124 124 205 205 205 F F F
163192024-02-04T14:07:29.169ZMax gen: 151, Max flush: 261
163202024-02-04T14:07:29.170Z
163212024-02-04T14:07:29.170ZBegin loop 76
163222024-02-04T14:07:29.170ZDownstairs to restart: 2
163232024-02-04T14:07:29.182Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.207125157Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737}
163242024-02-04T14:07:29.182Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.207608651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737}
163252024-02-04T14:07:29.183Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.207638863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737}
163262024-02-04T14:07:29.183Z{"msg":"Crucible 222941fa-27f2-4123-be51-e4ad3eec9b38 has session id: f82e9302-c2c1-4a23-b88d-15a48cc548be","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.207676587Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163272024-02-04T14:07:29.183Z{"msg":"Upstairs opts: Upstairs UUID: 222941fa-27f2-4123-be51-e4ad3eec9b38, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.207728464Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163282024-02-04T14:07:29.183Z{"msg":"Crucible runtime is spawned
163292024-02-04T14:07:29.183ZCrucible stats registered with UUID: 222941fa-27f2-4123-be51-e4ad3eec9b38","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.207841308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163302024-02-04T14:07:29.183Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.207932069Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737}
163312024-02-04T14:07:29.183Z{"msg":"222941fa-27f2-4123-be51-e4ad3eec9b38 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208204105Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163322024-02-04T14:07:29.183Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208263407Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163332024-02-04T14:07:29.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208295967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163342024-02-04T14:07:29.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208337397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163352024-02-04T14:07:29.184Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.20842199Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163362024-02-04T14:07:29.184Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208476462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163372024-02-04T14:07:29.198Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208522062Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163382024-02-04T14:07:29.198Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208589619Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"io task","client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163392024-02-04T14:07:29.198Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208635897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"io task","client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163402024-02-04T14:07:29.198Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208680263Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"io task","client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163412024-02-04T14:07:29.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.208903728Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"io task","client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163422024-02-04T14:07:29.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.209021518Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"io task","client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163432024-02-04T14:07:29.199Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.209072085Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"io task","client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163442024-02-04T14:07:29.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.2101735Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163452024-02-04T14:07:29.199Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.210264955Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163462024-02-04T14:07:29.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.210309587Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163472024-02-04T14:07:29.199Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.210350331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163482024-02-04T14:07:29.199Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.211239883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163492024-02-04T14:07:29.199Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.211311666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163502024-02-04T14:07:29.199Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.21135804Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163512024-02-04T14:07:29.200Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.211393967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163522024-02-04T14:07:29.200Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.212297188Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163532024-02-04T14:07:29.200Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.212340123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163542024-02-04T14:07:29.200Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.212364279Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163552024-02-04T14:07:29.200Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.212511476Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163562024-02-04T14:07:29.200Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.212543287Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163572024-02-04T14:07:29.200Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.212566545Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163582024-02-04T14:07:29.200Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.212625254Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163592024-02-04T14:07:29.200Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213354202Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163602024-02-04T14:07:29.200Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213412769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163612024-02-04T14:07:29.201Z{"msg":"[0]R flush_numbers[0..12]: [252, 252, 260, 260, 251, 247, 260, 239, 239, 260, 260, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213439094Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163622024-02-04T14:07:29.201Z{"msg":"[0]R generation[0..12]: [147, 147, 150, 150, 146, 144, 150, 141, 141, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213481646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163632024-02-04T14:07:29.201Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213507177Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163642024-02-04T14:07:29.201Z{"msg":"[1]R flush_numbers[0..12]: [252, 252, 260, 260, 251, 247, 260, 239, 239, 260, 260, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213549761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163652024-02-04T14:07:29.201Z{"msg":"[1]R generation[0..12]: [147, 147, 150, 150, 146, 144, 150, 141, 141, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213574432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163662024-02-04T14:07:29.201Z{"msg"Wait for a query_work_queue command to finish before sending IO
163672024-02-04T14:07:29.202Z:"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213621316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163682024-02-04T14:07:29.202Z{"msg":"[2]R flush_numbers[0..12]: [252, 252, 260, 260, 251, 247, 260, 239, 239, 260, 260, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213646099Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163692024-02-04T14:07:29.202Z{"msg":"[2]R generation[0..12]: [147, 147, 150, 150, 146, 144, 150, 141, 141, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213704747Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163702024-02-04T14:07:29.202Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213737448Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163712024-02-04T14:07:29.202Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213780988Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163722024-02-04T14:07:29.203Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213814739Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163732024-02-04T14:07:29.203Z{"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213879838Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163742024-02-04T14:07:29.203Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213906594Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"":"downstairs","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163752024-02-04T14:07:29.203Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213954289Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163762024-02-04T14:07:29.203Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.213975934Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"0","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163772024-02-04T14:07:29.204Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.214029567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"1","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163782024-02-04T14:07:29.204Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.214095854Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"client":"2","":"downstairs","session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163792024-02-04T14:07:29.204Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.214123031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163802024-02-04T14:07:29.204Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.214188565Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163812024-02-04T14:07:29.204Z{"msg":"222941fa-27f2-4123-be51-e4ad3eec9b38 is now active with session: f82e9302-c2c1-4a23-b88d-15a48cc548be","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.21421684Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163822024-02-04T14:07:29.204Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.214270318Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737,"session_id":"f82e9302-c2c1-4a23-b88d-15a48cc548be"}
163832024-02-04T14:07:29.204Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:28.214303113Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163842024-02-04T14:07:29.205Zip-10-150-1-226.us-west-2.compute.internal","pid":1737}
163852024-02-04T14:07:29.205ZLoading write count information from file "/tmp/test_repair_verify.data"
163862024-02-04T14:07:29.205ZRun Repair workload
163872024-02-04T14:07:29.205Z01/30 Write block 113 len 12288 data: 15 17 17
163882024-02-04T14:07:29.205Z02/30 Write block 446 len 16384 data: 17 16 15 13
163892024-02-04T14:07:29.206Z03/30 Write block 465 len 40960 data: 14 16 15 12 13 11 9 10 10 9
163902024-02-04T14:07:29.206Z04/30 Write block 51 len 12288 data: 8 9 9
163912024-02-04T14:07:29.206Z05/30 Read block 136 len 32768
163922024-02-04T14:07:29.222Z06/30 Write block 518 len 24576 data: 10 10 6 8 11 12
163932024-02-04T14:07:29.222Z07/30 Read block 147 len 4096
163942024-02-04T14:07:29.222Z08/30 Flush
163952024-02-04T14:07:29.223Z09/30 Write block 517 len 8192 data: 11 11
163962024-02-04T14:07:29.223Z10/30 Read block 437 len 12288
163972024-02-04T14:07:29.223Z11/30 Write block 288 len 28672 data: 15 15 14 8 10 7 5
163982024-02-04T14:07:29.254Z12/30 Write block 457 len 32768 data: 15 15 15 14 13 10 11 11
163992024-02-04T14:07:29.254Z13/30 Read block 20 len 16384
164002024-02-04T14:07:29.254Z14/30 Read block 239 len 28672
164012024-02-04T14:07:29.254Z15/30 Read block 199 len 12288
164022024-02-04T14:07:29.255Z16/30 Read block 278 len 20480
164032024-02-04T14:07:29.255Z17/30 Flush
164042024-02-04T14:07:29.255Z18/30 Write block 51 len 40960 data: 9 10 10 9 9 11 12 10 12 14
164052024-02-04T14:07:29.255Z19/30 Flush
164062024-02-04T14:07:29.256Z20/30 Write block 366 len 32768 data: 12 14 12 11 14 11 7 7
164072024-02-04T14:07:29.256Z21/30 Read block 90 len 12288
164082024-02-04T14:07:29.256Z22/30 Read block 196 len 12288
164092024-02-04T14:07:29.256Z23/30 Write block 136 len 40960 data: 7 6 7 9 13 9 5 6 9 10
164102024-02-04T14:07:29.256Z24/30 Read block 182 len 4096
164112024-02-04T14:07:29.256Z25/30 Flush
164122024-02-04T14:07:29.257Z26/30 Write block 270 len 12288 data: 11 10 7
164132024-02-04T14:07:29.257Z27/30 Read block 3 len 40960
164142024-02-04T14:07:29.257Z28/30 Write block 250 len 20480 data: 17 16 14 11 13
164152024-02-04T14:07:29.257Z29/30 Read block 319 len 12288
164162024-02-04T14:07:29.265Z30/30 Write block 157 len 28672 data: 7 8 9 9 9 10 13
164172024-02-04T14:07:29.266Z----------------------------------------------------------------
164182024-02-04T14:07:29.266Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:22
164192024-02-04T14:07:29.266ZStates: Active Active Active
164202024-02-04T14:07:29.267ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164212024-02-04T14:07:29.267Z 9 Acked 1008 Write 2 Done Done Done false
164222024-02-04T14:07:29.267Z 10 Acked 1009 Read 3 Done Done Sent false
164232024-02-04T14:07:29.268Z 11 Acked 1010 Write 7 Done Done Sent false
164242024-02-04T14:07:29.269Z 12 Acked 1011 Write 8 Done Done Sent false
164252024-02-04T14:07:29.269Z 13 Acked 1012 Read 4 Done Done Sent false
164262024-02-04T14:07:29.269Z 14 Acked 1013 Read 7 Done Done Sent false
164272024-02-04T14:07:29.270Z 15 Acked 1014 Read 3 Done Done Sent false
164282024-02-04T14:07:29.270Z 16 Acked 1015 Read 5 Done Done Sent false
164292024-02-04T14:07:29.270Z 17 Acked 1016 Flush 0 Done Done Sent false
164302024-02-04T14:07:29.270Z 18 Acked 1017 Write 10 Done Done Sent false
164312024-02-04T14:07:29.271Z 19 Acked 1018 Flush 0 Done Done Sent false
164322024-02-04T14:07:29.272Z 20 Acked 1019 Write 8 Done Done Sent false
164332024-02-04T14:07:29.272Z 21 Acked 1020 Read 3 Done Done Sent false
164342024-02-04T14:07:29.272Z 22 Acked 1021 Read 3 Done Done Sent false
164352024-02-04T14:07:29.273Z 23 Acked 1022 Write 10 Done Done Sent false
164362024-02-04T14:07:29.273Z 24 Acked 1023 Read 1 Done Done Sent false
164372024-02-04T14:07:29.273Z 25 Acked 1024 Flush 0 Done Done Sent false
164382024-02-04T14:07:29.273Z 26 Acked 1025 Write 3 Done Done Sent false
164392024-02-04T14:07:29.274Z 27 Acked 1026 Read 10 Done Done Sent false
164402024-02-04T14:07:29.274Z 28 Acked 1027 Write 5 Done Done Sent false
164412024-02-04T14:07:29.274Z 29 Acked 1028 Read 3 Done Done Sent false
164422024-02-04T14:07:29.274Z 30 Acked 1029 Write 7 Sent Sent Sent false
164432024-02-04T14:07:29.274Z STATES DS:0 DS:1 DS:2 TOTAL
164442024-02-04T14:07:29.275Z New 0 0 0 0
164452024-02-04T14:07:29.275Z Sent 1 1 21 23
164462024-02-04T14:07:29.275Z Done 21 21 1 43
164472024-02-04T14:07:29.275Z Skipped 0 0 0 0
164482024-02-04T14:07:29.276Z Error 0 0 0 0
164492024-02-04T14:07:29.276ZLast Flush: 1024 1024 1007
164502024-02-04T14:07:29.276ZDownstairs last five completed: 1007 1006 1005 1004 1003
164512024-02-04T14:07:29.276ZUpstairs last five completed: 30 29 28 27 26
164522024-02-04T14:07:29.277Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:28.26462269Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1737}
164532024-02-04T14:07:29.277ZWrote out file "/tmp/test_repair_verify.data"
164542024-02-04T14:07:29.277Z
164552024-02-04T14:07:29.277ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164562024-02-04T14:07:29.284ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164572024-02-04T14:07:29.284Z 0 000-019 147 147 147 252 252 252 F F F
164582024-02-04T14:07:29.284Z 1 020-039 147 147 147 252 252 252 F F F
164592024-02-04T14:07:29.285Z 2 040-059 152 152 152 264 264 262 F F F <---
164602024-02-04T14:07:29.285Z 3 060-079 152 152 150 264 264 260 F F F <---
164612024-02-04T14:07:29.285Z 4 080-099 146 146 146 251 251 251 F F F
164622024-02-04T14:07:29.285Z 5 100-119 152 152 152 262 262 262 F F F
164632024-02-04T14:07:29.286Z 6 120-139 152 152 150 265 265 260 F F F <---
164642024-02-04T14:07:29.286Z 7 140-159 152 152 141 265 265 239 T T F <---
164652024-02-04T14:07:29.286Z 8 160-179 141 141 141 239 239 239 T T F <---
164662024-02-04T14:07:29.286Z 9 180-199 150 150 150 260 260 260 F F F
164672024-02-04T14:07:29.287Z 10 200-219 150 150 150 260 260 260 F F F
164682024-02-04T14:07:29.287Z 11 220-239 148 148 148 254 254 254 F F F
164692024-02-04T14:07:29.287Z 12 240-259 151 151 151 261 261 261 T T F <---
164702024-02-04T14:07:29.287Z 13 260-279 150 150 150 260 260 260 T T F <---
164712024-02-04T14:07:29.287Z 14 280-299 152 152 151 263 263 261 F F F <---
164722024-02-04T14:07:29.288Z 15 300-319 150 150 150 260 260 260 F F F
164732024-02-04T14:07:29.288Z 16 320-339 150 150 150 259 259 259 F F F
164742024-02-04T14:07:29.288Z 17 340-359 149 149 149 258 258 258 F F F
164752024-02-04T14:07:29.288Z 18 360-379 152 152 144 265 265 247 F F F <---
164762024-02-04T14:07:29.288Z 19 380-399 144 144 144 245 245 245 F F F
164772024-02-04T14:07:29.289Z 20 400-419 148 148 148 257 257 257 F F F
164782024-02-04T14:07:29.289Z 21 420-439 149 149 149 258 258 258 F F F
164792024-02-04T14:07:29.289Z 22 440-459 152 152 152 263 263 262 F F F <---
164802024-02-04T14:07:29.289Z 23 460-479 152 152 152 263 263 262 F F F <---
164812024-02-04T14:07:29.289Z 24 480-499 142 142 142 241 241 241 F F F
164822024-02-04T14:07:29.290Z 25 500-519 152 152 152 263 263 262 F F T <---
164832024-02-04T14:07:29.290Z 26 520-539 152 152 152 262 262 262 F F F
164842024-02-04T14:07:29.290Z 27 540-559 148 148 148 256 256 256 F F F
164852024-02-04T14:07:29.290Z 28 560-579 150 150 150 260 260 260 F F F
164862024-02-04T14:07:29.290Z 29 580-599 124 124 124 205 205 205 F F F
164872024-02-04T14:07:29.291ZMax gen: 152, Max flush: 265
164882024-02-04T14:07:29.291ZError: Difference in extent metadata found!
164892024-02-04T14:07:29.291ZA Difference in extent metadata is expected here
164902024-02-04T14:07:29.291ZOn loop 76
164912024-02-04T14:07:29.291Z
164922024-02-04T14:07:29.292ZVerifying data now
164932024-02-04T14:07:30.341ZNo change after verify
164942024-02-04T14:07:30.342ZLoop: 76 Downstairs dump after verify (and repair):
164952024-02-04T14:07:30.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164962024-02-04T14:07:30.366Z 0 000-019 147 147 147 252 252 252 F F F
164972024-02-04T14:07:30.367Z 1 020-039 147 147 147 252 252 252 F F F
164982024-02-04T14:07:30.367Z 2 040-059 152 152 152 264 264 264 F F F
164992024-02-04T14:07:30.367Z 3 060-079 152 152 152 264 264 264 F F F
165002024-02-04T14:07:30.368Z 4 080-099 146 146 146 251 251 251 F F F
165012024-02-04T14:07:30.368Z 5 100-119 152 152 152 262 262 262 F F F
165022024-02-04T14:07:30.368Z 6 120-139 152 152 152 265 265 265 F F F
165032024-02-04T14:07:30.368Z 7 140-159 153 153 153 266 266 266 F F F
165042024-02-04T14:07:30.379Z 8 160-179 153 153 153 266 266 266 F F F
165052024-02-04T14:07:30.380Z 9 180-199 150 150 150 260 260 260 F F F
165062024-02-04T14:07:30.380Z 10 200-219 150 150 150 260 260 260 F F F
165072024-02-04T14:07:30.380Z 11 220-239 148 148 148 254 254 254 F F F
165082024-02-04T14:07:30.380Z 12 240-259 153 153 153 266 266 266 F F F
165092024-02-04T14:07:30.381Z 13 260-279 153 153 153 266 266 266 F F F
165102024-02-04T14:07:30.381Z 14 280-299 152 152 152 263 263 263 F F F
165112024-02-04T14:07:30.381Z 15 300-319 150 150 150 260 260 260 F F F
165122024-02-04T14:07:30.381Z 16 320-339 150 150 150 259 259 259 F F F
165132024-02-04T14:07:30.381Z 17 340-359 149 149 149 258 258 258 F F F
165142024-02-04T14:07:30.382Z 18 360-379 152 152 152 265 265 265 F F F
165152024-02-04T14:07:30.382Z 19 380-399 144 144 144 245 245 245 F F F
165162024-02-04T14:07:30.382Z 20 400-419 148 148 148 257 257 257 F F F
165172024-02-04T14:07:30.382Z 21 420-439 149 149 149 258 258 258 F F F
165182024-02-04T14:07:30.382Z 22 440-459 152 152 152 263 263 263 F F F
165192024-02-04T14:07:30.383Z 23 460-479 152 152 152 263 263 263 F F F
165202024-02-04T14:07:30.383Z 24 480-499 142 142 142 241 241 241 F F F
165212024-02-04T14:07:30.383Z 25 500-519 152 152 152 263 263 263 F F F
165222024-02-04T14:07:30.383Z 26 520-539 152 152 152 262 262 262 F F F
165232024-02-04T14:07:30.383Z 27 540-559 148 148 148 256 256 256 F F F
165242024-02-04T14:07:30.383Z 28 560-579 150 150 150 260 260 260 F F F
165252024-02-04T14:07:30.384Z 29 580-599 124 124 124 205 205 205 F F F
165262024-02-04T14:07:30.384ZMax gen: 153, Max flush: 266
165272024-02-04T14:07:30.384Z
165282024-02-04T14:07:30.384ZBegin loop 77
165292024-02-04T14:07:30.384ZDownstairs to restart: 1
165302024-02-04T14:07:30.396Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.426633209Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745}
165312024-02-04T14:07:30.396Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427063037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745}
165322024-02-04T14:07:30.405Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427193724Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745}
165332024-02-04T14:07:30.406Z{"msg":"Crucible 0aa50fb1-818e-4d69-b519-c2bd3712c93a has session id: 4f1aadee-6da4-450f-b960-71f504b6c0c8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.42731619Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"Crucible runtime is spawned
165342024-02-04T14:07:30.406Zsession_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165352024-02-04T14:07:30.406Z{"msg":"Upstairs opts: Upstairs UUID: 0aa50fb1-818e-4d69-b519-c2bd3712c93a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427546776Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165362024-02-04T14:07:30.406Z{"msg":"Crucible stats registered with UUID: 0aa50fb1-818e-4d69-b519-c2bd3712c93a","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427635866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165372024-02-04T14:07:30.406Z{"msg":"0aa50fb1-818e-4d69-b519-c2bd3712c93a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427771075Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165382024-02-04T14:07:30.407Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427843947Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745}
165392024-02-04T14:07:30.407Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427866951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165402024-02-04T14:07:30.407Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427891753Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165412024-02-04T14:07:30.407Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427932149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165422024-02-04T14:07:30.407Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.427987755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165432024-02-04T14:07:30.408Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.428014067Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165442024-02-04T14:07:30.408Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.428074194Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165452024-02-04T14:07:30.408Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.428106371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"io task","client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165462024-02-04T14:07:30.408Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.428300299Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"io task","client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165472024-02-04T14:07:30.408Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.428374063Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"io task","client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165482024-02-04T14:07:30.409Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.428429675Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"io task","client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165492024-02-04T14:07:30.409Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.428474365Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"io task","client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165502024-02-04T14:07:30.409Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.428540934Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"io task","client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165512024-02-04T14:07:30.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.429472223Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165522024-02-04T14:07:30.409Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.429541607Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165532024-02-04T14:07:30.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.429567946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165542024-02-04T14:07:30.409Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.429590534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165552024-02-04T14:07:30.409Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.430689141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165562024-02-04T14:07:30.409Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.430931002Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165572024-02-04T14:07:30.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.431542193Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165582024-02-04T14:07:30.410Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.431637989Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165592024-02-04T14:07:30.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.431774969Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165602024-02-04T14:07:30.410Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.43183435Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165612024-02-04T14:07:30.410Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.431859651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165622024-02-04T14:07:30.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.431880635Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165632024-02-04T14:07:30.410Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.431921324Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165642024-02-04T14:07:30.411Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.431944693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165652024-02-04T14:07:30.411Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.432466975Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165662024-02-04T14:07:30.419Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.43314837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165672024-02-04T14:07:30.419Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433236538Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165682024-02-04T14:07:30.420Z{"msg":"[0]R flush_numbers[0..12]: [252, 252, 264, 264, 251, 262, 265, 266, 266, 260, 260, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433306769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165692024-02-04T14:07:30.420Z{"msg":"[0]R generation[0..12]: [147, 147, 152, 152, 146, 152, 152, 153, 153, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433335311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165702024-02-04T14:07:30.420Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433406056Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165712024-02-04T14:07:30.420Z{"msg":"[1]R flush_numbers[0..12]: [252, 252, 264, 264, 251, 262, 265, 266, 266, 260, 260, 254]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
165722024-02-04T14:07:30.421Z,"time":"2024-02-04T14:07:29.433488231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165732024-02-04T14:07:30.421Z{"msg":"[1]R generation[0..12]: [147, 147, 152, 152, 146, 152, 152, 153, 153, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433557117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165742024-02-04T14:07:30.421Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.43359195Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165752024-02-04T14:07:30.421Z{"msg":"[2]R flush_numbers[0..12]: [252, 252, 264, 264, 251, 262, 265, 266, 266, 260, 260, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433648468Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165762024-02-04T14:07:30.421Z{"msg":"[2]R generation[0..12]: [147, 147, 152, 152, 146, 152, 152, 153, 153, 150, 150, 148]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433675267Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165772024-02-04T14:07:30.421Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433748363Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165782024-02-04T14:07:30.422Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.43382458Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165792024-02-04T14:07:30.422Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433887863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165802024-02-04T14:07:30.422Z{"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433916466Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165812024-02-04T14:07:30.422Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.433980296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"":"downstairs","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165822024-02-04T14:07:30.422Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.434006974Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165832024-02-04T14:07:30.423Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.434034901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"0","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165842024-02-04T14:07:30.423Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.434126142Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"1","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165852024-02-04T14:07:30.423Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.434168154Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"client":"2","":"downstairs","session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165862024-02-04T14:07:30.423Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.434209575Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165872024-02-04T14:07:30.424Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.434274141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165882024-02-04T14:07:30.424Z{"msg":"0aa50fb1-818e-4d69-b519-c2bd3712c93a is now active with session: 4f1aadee-6da4-450f-b960-71f504b6c0c8","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165892024-02-04T14:07:30.424Z,"time":"2024-02-04T14:07:29.434314401Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165902024-02-04T14:07:30.424Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.434360134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745,"session_id":"4f1aadee-6da4-450f-b960-71f504b6c0c8"}
165912024-02-04T14:07:30.424Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:29.43441838Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745}
165922024-02-04T14:07:30.425ZLoading write count information from file "/tmp/test_repair_verify.data"
165932024-02-04T14:07:30.425ZRun Repair workload
165942024-02-04T14:07:30.425Z01/30 Write block 189 len 8192 data: 12 10
165952024-02-04T14:07:30.425Z02/30 Read block 43 len 20480
165962024-02-04T14:07:30.425Z03/30 Read block 385 len 28672
165972024-02-04T14:07:30.426Z04/30 Read block 320 len 40960
165982024-02-04T14:07:30.426Z05/30 Read block 64 len 32768
165992024-02-04T14:07:30.426Z06/30 Write block 34 len 24576 data: 11 14 15 13 11 10
166002024-02-04T14:07:30.426Z07/30 Read block 105 len 36864
166012024-02-04T14:07:30.426Z08/30 Write block 161 len 8192 data: 10 11
166022024-02-04T14:07:30.427Z09/30 Read block 501 len 32768
166032024-02-04T14:07:30.432Z10/30 Read block 393 len 28672
166042024-02-04T14:07:30.432Z11/30 Write block 228 len 36864 data: 13 11 10 9 9 11 14 11 13
166052024-02-04T14:07:30.433Z12/30 Read block 263 len 24576
166062024-02-04T14:07:30.433Z13/30 Read block 556 len 16384
166072024-02-04T14:07:30.433Z14/30 Write block 478 len 40960 data: 9 9 7 6 4 7 9 11 9 8
166082024-02-04T14:07:30.433Z15/30 Read block 131 len 40960
166092024-02-04T14:07:30.433Z16/30 Read block 412 len 36864
166102024-02-04T14:07:30.434Z17/30 Write block 184 len 12288 data: 17 17 15
166112024-02-04T14:07:30.434Z18/30 Read block 210 len 36864
166122024-02-04T14:07:30.439Z19/30 Write block 185 len 16384 data: 18 16 15 12
166132024-02-04T14:07:30.439Z20/30 Read block 253 len 24576
166142024-02-04T14:07:30.439Z21/30 Read block 113 len 28672
166152024-02-04T14:07:30.440Z22/30 Read block 107 len 16384
166162024-02-04T14:07:30.440Z23/30 Read block 305 len 16384
166172024-02-04T14:07:30.445Z24/30 Read block 152 len 40960
166182024-02-04T14:07:30.446Z25/30 Read block 473 len 24576
166192024-02-04T14:07:30.446Z26/30 Write block 382 len 12288 data: 7 9 10
166202024-02-04T14:07:30.446Z27/30 Write block 209 len 16384 data: 9 9 11 11
166212024-02-04T14:07:30.446Z28/30 Read block 171 len 32768
166222024-02-04T14:07:30.452Z29/30 Read block 225 len 24576
166232024-02-04T14:07:30.452Z30/30 Read block 330 len 4096
166242024-02-04T14:07:30.452Z----------------------------------------------------------------
166252024-02-04T14:07:30.453Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166262024-02-04T14:07:30.453ZStates: Active Active Active
166272024-02-04T14:07:30.453ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166282024-02-04T14:07:30.453Z 1 Acked 1000 Write 2 Done Done Done false
166292024-02-04T14:07:30.453Z 2 Acked 1001 Read 5 Done Done Done false
166302024-02-04T14:07:30.454Z 3 Acked 1002 Read 7 Done Sent Done false
166312024-02-04T14:07:30.454Z 4 Acked 1003 Read 10 Done Sent Done false
166322024-02-04T14:07:30.454Z 5 Acked 1004 Read 8 Done Sent Done false
166332024-02-04T14:07:30.454Z 6 Acked 1005 Write 6 Done Sent Done false
166342024-02-04T14:07:30.454Z 7 Acked 1006 Read 9 Done Sent Done false
166352024-02-04T14:07:30.454Z 8 Acked 1007 Write 2 Done Sent Done false
166362024-02-04T14:07:30.455Z 9 Acked 1008 Read 8 Done Sent Done false
166372024-02-04T14:07:30.455Z 10 Acked 1009 Read 7 Done Sent Done false
166382024-02-04T14:07:30.455Z 11 Acked 1010 Write 9 Done Sent Done false
166392024-02-04T14:07:30.455Z 12 Acked 1011 Read 6 Done Sent Done false
166402024-02-04T14:07:30.456Z 13 Acked 1012 Read 4 Done Sent Done false
166412024-02-04T14:07:30.456Z 14 Acked 1013 Write 10 Done Sent Done false
166422024-02-04T14:07:30.456Z 15 Acked 1014 Read 10 Done Sent Done false
166432024-02-04T14:07:30.456Z 16 Acked 1015 Read 9 Done Sent Done false
166442024-02-04T14:07:30.456Z 17 Acked 1016 Write 3 Done Sent Done false
166452024-02-04T14:07:30.456Z 18 Acked 1017 Read 9 Done Sent Done false
166462024-02-04T14:07:30.457Z 19 Acked 1018 Write 4 Done Sent Done false
166472024-02-04T14:07:30.457Z 20 Acked 1019 Read 6 Done Sent Done false
166482024-02-04T14:07:30.457Z 21 Acked 1020 Read 7 Done Sent Done false
166492024-02-04T14:07:30.457Z 22 Acked 1021 Read 4 Done Sent Done false
166502024-02-04T14:07:30.457Z 23 Acked 1022 Read 4 Done Sent Done false
166512024-02-04T14:07:30.458Z 24 Acked 1023 Read 10 Done Sent Done false
166522024-02-04T14:07:30.458Z 25 Acked 1024 Read 6 Done Sent Done false
166532024-02-04T14:07:30.458Z 26 Acked 1025 Write 3 Done Sent Done false
166542024-02-04T14:07:30.458Z 27 Acked 1026 Write 4 Done Sent Done false
166552024-02-04T14:07:30.458Z 28 Acked 1027 Read 8 Done Sent Done false
166562024-02-04T14:07:30.458Z 29 Acked 1028 Read 6 Done Sent Done false
166572024-02-04T14:07:30.459Z 30 Acked 1029 Read 1 Done Sent Done false
166582024-02-04T14:07:30.459Z STATES DS:0 DS:1 DS:2 TOTAL
166592024-02-04T14:07:30.459Z New 0 0 0 0
166602024-02-04T14:07:30.459Z Sent 0 28 0 28
166612024-02-04T14:07:30.460Z Done 30 2 30 62
166622024-02-04T14:07:30.460Z Skipped 0 0 0 0
166632024-02-04T14:07:30.460Z Error 0 0 0 0
166642024-02-04T14:07:30.460ZLast Flush: 0 0 0
166652024-02-04T14:07:30.460ZDownstairs last five completed:
166662024-02-04T14:07:30.461ZUpstairs last five completed: 30 29 28 27 26
166672024-02-04T14:07:30.461Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:29.480581114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1745}
166682024-02-04T14:07:30.461ZWrote out file "/tmp/test_repair_verify.data"
166692024-02-04T14:07:30.469Z
166702024-02-04T14:07:30.469ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166712024-02-04T14:07:30.490ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166722024-02-04T14:07:30.491Z 0 000-019 147 147 147 252 252 252 F F F
166732024-02-04T14:07:30.491Z 1 020-039 147 147 147 252 252 252 T F T <---
166742024-02-04T14:07:30.491Z 2 040-059 152 152 152 264 264 264 F F F
166752024-02-04T14:07:30.491Z 3 060-079 152 152 152 264 264 264 F F F
166762024-02-04T14:07:30.491Z 4 080-099 146 146 146 251 251 251 F F F
166772024-02-04T14:07:30.492Z 5 100-119 152 152 152 262 262 262 F F F
166782024-02-04T14:07:30.492Z 6 120-139 152 152 152 265 265 265 F F F
166792024-02-04T14:07:30.492Z 7 140-159 153 153 153 266 266 266 F F F
166802024-02-04T14:07:30.492Z 8 160-179 153 153 153 266 266 266 T F T <---
166812024-02-04T14:07:30.498Z 9 180-199 150 150 150 260 260 260 T T T
166822024-02-04T14:07:30.499Z 10 200-219 150 150 150 260 260 260 T F T <---
166832024-02-04T14:07:30.499Z 11 220-239 148 148 148 254 254 254 T F T <---
166842024-02-04T14:07:30.499Z 12 240-259 153 153 153 266 266 266 F F F
166852024-02-04T14:07:30.499Z 13 260-279 153 153 153 266 266 266 F F F
166862024-02-04T14:07:30.500Z 14 280-299 152 152 152 263 263 263 F F F
166872024-02-04T14:07:30.500Z 15 300-319 150 150 150 260 260 260 F F F
166882024-02-04T14:07:30.500Z 16 320-339 150 150 150 259 259 259 F F F
166892024-02-04T14:07:30.500Z 17 340-359 149 149 149 258 258 258 F F F
166902024-02-04T14:07:30.500Z 18 360-379 152 152 152 265 265 265 F F F
166912024-02-04T14:07:30.501Z 19 380-399 144 144 144 245 245 245 T F T <---
166922024-02-04T14:07:30.501Z 20 400-419 148 148 148 257 257 257 F F F
166932024-02-04T14:07:30.501Z 21 420-439 149 149 149 258 258 258 F F F
166942024-02-04T14:07:30.501Z 22 440-459 152 152 152 263 263 263 F F F
166952024-02-04T14:07:30.501Z 23 460-479 152 152 152 263 263 263 T F T <---
166962024-02-04T14:07:30.501Z 24 480-499 142 142 142 241 241 241 T F T <---
166972024-02-04T14:07:30.502Z 25 500-519 152 152 152 263 263 263 F F F
166982024-02-04T14:07:30.502Z 26 520-539 152 152 152 262 262 262 F F F
166992024-02-04T14:07:30.502Z 27 540-559 148 148 148 256 256 256 F F F
167002024-02-04T14:07:30.502Z 28 560-579 150 150 150 260 260 260 F F F
167012024-02-04T14:07:30.502Z 29 580-599 124 124 124 205 205 205 F F F
167022024-02-04T14:07:30.502ZMax gen: 153, Max flush: 266
167032024-02-04T14:07:30.503ZError: Difference in extent metadata found!
167042024-02-04T14:07:30.503ZA Difference in extent metadata is expected here
167052024-02-04T14:07:30.503ZOn loop 77
167062024-02-04T14:07:30.503Z
167072024-02-04T14:07:30.509ZVerifying data now
167082024-02-04T14:07:35.284ZNo change after verify
167092024-02-04T14:07:35.284ZLoop: 77 Downstairs dump after verify (and repair):
167102024-02-04T14:07:35.306ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167112024-02-04T14:07:35.307Z 0 000-019 147 147 147 252 252 252 F F F
167122024-02-04T14:07:35.307Z 1 020-039 154 154 154 267 267 267 F F F
167132024-02-04T14:07:35.307Z 2 040-059 152 152 152 264 264 264 F F F
167142024-02-04T14:07:35.307Z 3 060-079 152 152 152 264 264 264 F F F
167152024-02-04T14:07:35.307Z 4 080-099 146 146 146 251 251 251 F F F
167162024-02-04T14:07:35.307Z 5 100-119 152 152 152 262 262 262 F F F
167172024-02-04T14:07:35.307Z 6 120-139 152 152 152 265 265 265 F F F
167182024-02-04T14:07:35.307Z 7 140-159 153 153 153 266 266 266 F F F
167192024-02-04T14:07:35.308Z 8 160-179 154 154 154 267 267 267 F F F
167202024-02-04T14:07:35.308Z 9 180-199 154 154 154 267 267 267 F F F
167212024-02-04T14:07:35.336Z 10 200-219 154 154 154 267 267 267 F F F
167222024-02-04T14:07:35.336Z 11 220-239 154 154 154 267 267 267 F F F
167232024-02-04T14:07:35.336Z 12 240-259 153 153 153 266 266 266 F F F
167242024-02-04T14:07:35.337Z 13 260-279 153 153 153 266 266 266 F F F
167252024-02-04T14:07:35.337Z 14 280-299 152 152 152 263 263 263 F F F
167262024-02-04T14:07:35.337Z 15 300-319 150 150 150 260 260 260 F F F
167272024-02-04T14:07:35.337Z 16 320-339 150 150 150 259 259 259 F F F
167282024-02-04T14:07:35.337Z 17 340-359 149 149 149 258 258 258 F F F
167292024-02-04T14:07:35.337Z 18 360-379 152 152 152 265 265 265 F F F
167302024-02-04T14:07:35.337Z 19 380-399 154 154 154 267 267 267 F F F
167312024-02-04T14:07:35.337Z 20 400-419 148 148 148 257 257 257 F F F
167322024-02-04T14:07:35.338Z 21 420-439 149 149 149 258 258 258 F F F
167332024-02-04T14:07:35.338Z 22 440-459 152 152 152 263 263 263 F F F
167342024-02-04T14:07:35.338Z 23 460-479 154 154 154 267 267 267 F F F
167352024-02-04T14:07:35.338Z 24 480-499 154 154 154 267 267 267 F F F
167362024-02-04T14:07:35.338Z 25 500-519 152 152 152 263 263 263 F F F
167372024-02-04T14:07:35.338Z 26 520-539 152 152 152 262 262 262 F F F
167382024-02-04T14:07:35.338Z 27 540-559 148 148 148 256 256 256 F F F
167392024-02-04T14:07:35.338Z 28 560-579 150 150 150 260 260 260 F F F
167402024-02-04T14:07:35.339Z 29 580-599 124 124 124 205 205 205 F F F
167412024-02-04T14:07:35.339ZMax gen: 154, Max flush: 267
167422024-02-04T14:07:35.339Z
167432024-02-04T14:07:35.339ZBegin loop 78
167442024-02-04T14:07:35.339ZDownstairs to restart: 1
167452024-02-04T14:07:35.376Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.366495416Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753}
167462024-02-04T14:07:35.376Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367008801Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753}
167472024-02-04T14:07:35.376Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367093022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753}
167482024-02-04T14:07:35.376ZCrucible runtime is spawned
167492024-02-04T14:07:35.376Z{"msg":"Crucible 312c590c-bd38-49c1-b771-628b79702e9c has session id: b1de7874-4d9a-49b0-930c-b1bc3a297b3b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367220953Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167502024-02-04T14:07:35.376Z{"msg":"Upstairs opts: Upstairs UUID: 312c590c-bd38-49c1-b771-628b79702e9c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367347748Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167512024-02-04T14:07:35.376Z{"msg":"Crucible stats registered with UUID: 312c590c-bd38-49c1-b771-628b79702e9c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367383573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167522024-02-04T14:07:35.376Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367405752Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753}
167532024-02-04T14:07:35.376Z{"msg":"312c590c-bd38-49c1-b771-628b79702e9c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.36742496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167542024-02-04T14:07:35.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367475298Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167552024-02-04T14:07:35.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367506937Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167562024-02-04T14:07:35.376Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367628891Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167572024-02-04T14:07:35.376Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.367700666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167582024-02-04T14:07:35.377Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.36773951Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167592024-02-04T14:07:35.377Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.36797335Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167602024-02-04T14:07:35.377Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.368056731Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"io task","client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167612024-02-04T14:07:35.377Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.368104971Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"io task","client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167622024-02-04T14:07:35.377Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.368181665Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"io task","client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167632024-02-04T14:07:35.377Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.368223795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"io task","client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167642024-02-04T14:07:35.377Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.368316744Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"io task","client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167652024-02-04T14:07:35.377Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.368362415Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"io task","client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167662024-02-04T14:07:35.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.36941712Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167672024-02-04T14:07:35.377Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.369495487Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167682024-02-04T14:07:35.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.369544653Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167692024-02-04T14:07:35.377Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.369588426Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167702024-02-04T14:07:35.378Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.370304283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167712024-02-04T14:07:35.378Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.370488611Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167722024-02-04T14:07:35.378Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.370755486Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167732024-02-04T14:07:35.378Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.370901218Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167742024-02-04T14:07:35.378Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.371241173Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167752024-02-04T14:07:35.378Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.371310935Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167762024-02-04T14:07:35.378Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.37133859Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167772024-02-04T14:07:35.378Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.371499519Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167782024-02-04T14:07:35.379Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.371557277Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167792024-02-04T14:07:35.379Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.371582033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167802024-02-04T14:07:35.379Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.371864331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167812024-02-04T14:07:35.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372550314Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167822024-02-04T14:07:35.379Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372624558Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167832024-02-04T14:07:35.379Z{"msg":"[0]R flush_numbers[0..12]: [252, 267, 264, 264, 251, 262, 265, 266, 267, 267, 267, 267]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372655997Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167842024-02-04T14:07:35.379Z{"msg":"[0]R generation[0..12]: [147, 154, 152, 152, 146, 152, 152, 153, 154, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372680438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167852024-02-04T14:07:35.380Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.37272055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167862024-02-04T14:07:35.380Z{"msg":"[1]R flush_numbers[0..12]: [252, 267, 264, 264, 251, 262, 265, 266, 267, 267, 267, 267]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372781078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167872024-02-04T14:07:35.380Z{"msg":"[1]R generation[0..12]: [147, 154, 152, 152, 146, 152, 152, 153, 154, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372809241Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167882024-02-04T14:07:35.380Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
167892024-02-04T14:07:35.380Z,"time":"2024-02-04T14:07:34.372853391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167902024-02-04T14:07:35.381Z{"msg":"[2]R flush_numbers[0..12]: [252, 267, 264, 264, 251, 262, 265, 266, 267, 267, 267, 267]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372879265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167912024-02-04T14:07:35.381Z{"msg":"[2]R generation[0..12]: [147, 154, 152, 152, 146, 152, 152, 153, 154, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372926322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167922024-02-04T14:07:35.381Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372951563Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167932024-02-04T14:07:35.381Z{"msg":"Max found gen is 155","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.372999795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167942024-02-04T14:07:35.381Z{"msg":"Generation requested: 156 >= found:155","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373023314Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167952024-02-04T14:07:35.381Z{"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.37306809Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167962024-02-04T14:07:35.381Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373091577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"":"downstairs","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167972024-02-04T14:07:35.382Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373134265Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167982024-02-04T14:07:35.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373156084Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"0","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
167992024-02-04T14:07:35.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373200107Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"1","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
168002024-02-04T14:07:35.382Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373223375Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"client":"2","":"downstairs","session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
168012024-02-04T14:07:35.382Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373264606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
168022024-02-04T14:07:35.382Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373287075Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
168032024-02-04T14:07:35.382Z{"msg":"312c590c-bd38-49c1-b771-628b79702e9c is now active with session: b1de7874-4d9a-49b0-930c-b1bc3a297b3b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373327807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
168042024-02-04T14:07:35.383Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373349152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753,"session_id":"b1de7874-4d9a-49b0-930c-b1bc3a297b3b"}
168052024-02-04T14:07:35.383Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:34.373391435Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753}
168062024-02-04T14:07:35.383ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
168072024-02-04T14:07:35.383ZLoading write count information from file "/tmp/test_repair_verify.data"
168082024-02-04T14:07:35.383ZRun Repair workload
168092024-02-04T14:07:35.383Z01/30 Write block 319 len 4096 data: 12
168102024-02-04T14:07:35.384Z02/30 Read block 384 len 40960
168112024-02-04T14:07:35.384Z03/30 Read block 178 len 36864
168122024-02-04T14:07:35.408Z04/30 Read block 593 len 16384
168132024-02-04T14:07:35.408Z05/30 Read block 52 len 4096
168142024-02-04T14:07:35.408Z06/30 Read block 311 len 12288
168152024-02-04T14:07:35.408Z07/30 Write block 231 len 8192 data: 10 10
168162024-02-04T14:07:35.408Z08/30 Read block 508 len 16384
168172024-02-04T14:07:35.408Z09/30 Flush
168182024-02-04T14:07:35.408Z10/30 Write block 374 len 12288 data: 11 11 8
168192024-02-04T14:07:35.408Z11/30 Write block 351 len 16384 data: 16 16 16 12
168202024-02-04T14:07:35.408Z12/30 Write block 140 len 24576 data: 14 10 6 7 10 11
168212024-02-04T14:07:35.408Z13/30 Write block 336 len 32768 data: 8 8 8 9 11 11 10 10
168222024-02-04T14:07:35.408Z14/30 Read block 147 len 8192
168232024-02-04T14:07:35.408Z15/30 Write block 334 len 32768 data: 12 11 9 9 9 10 12 12
168242024-02-04T14:07:35.408Z16/30 Read block 100 len 12288
168252024-02-04T14:07:35.408Z17/30 Flush
168262024-02-04T14:07:35.408Z18/30 Write block 240 len 4096 data: 15
168272024-02-04T14:07:35.409Z19/30 Read block 454 len 4096
168282024-02-04T14:07:35.409Z20/30 Read block 189 len 28672
168292024-02-04T14:07:35.409Z21/30 Read block 352 len 4096
168302024-02-04T14:07:35.409Z22/30 Read block 505 len 32768
168312024-02-04T14:07:35.409Z23/30 Flush
168322024-02-04T14:07:35.409Z24/30 Write block 174 len 16384 data: 13 13 13 17
168332024-02-04T14:07:35.409Z25/30 Read block 224 len 12288
168342024-02-04T14:07:35.409Z26/30 Flush
168352024-02-04T14:07:35.409Z27/30 Write block 236 len 24576 data: 14 17 14 13 16 14
168362024-02-04T14:07:35.409Z28/30 Read block 439 len 40960
168372024-02-04T14:07:35.409Z29/30 Read block 8 len 32768
168382024-02-04T14:07:35.410Z30/30 Read block 459 len 28672
168392024-02-04T14:07:35.410Z----------------------------------------------------------------
168402024-02-04T14:07:35.410Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
168412024-02-04T14:07:35.410ZStates: Active Active Active
168422024-02-04T14:07:35.410ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168432024-02-04T14:07:35.410Z 1 Acked 1000 Write 1 Done Done Done false
168442024-02-04T14:07:35.411Z 2 Acked 1001 Read 10 Done Done Done false
168452024-02-04T14:07:35.411Z 3 Acked 1002 Read 9 Done Done Done false
168462024-02-04T14:07:35.411Z 4 Acked 1003 Read 4 Done Done Done false
168472024-02-04T14:07:35.411Z 5 Acked 1004 Read 1 Done Done Done false
168482024-02-04T14:07:35.411Z 6 Acked 1005 Read 3 Done Done Done false
168492024-02-04T14:07:35.411Z 7 Acked 1006 Write 2 Done Done Done false
168502024-02-04T14:07:35.411Z 8 Acked 1007 Read 4 Done Done Done false
168512024-02-04T14:07:35.411Z 9 Acked 1008 Flush 0 Done Sent Done false
168522024-02-04T14:07:35.412Z 10 Acked 1009 Write 3 Done Sent Done false
168532024-02-04T14:07:35.412Z 11 Acked 1010 Write 4 Done Sent Done false
168542024-02-04T14:07:35.412Z 12 Acked 1011 Write 6 Done Sent Done false
168552024-02-04T14:07:35.412Z 13 Acked 1012 Write 8 Done Sent Done false
168562024-02-04T14:07:35.412Z 14 Acked 1013 Read 2 Done Sent Done false
168572024-02-04T14:07:35.412Z 15 Acked 1014 Write 8 Done Sent Done false
168582024-02-04T14:07:35.412Z 16 Acked 1015 Read 3 Done Sent Done false
168592024-02-04T14:07:35.413Z 17 Acked 1016 Flush 0 Done Sent Done false
168602024-02-04T14:07:35.413Z 18 Acked 1017 Write 1 Done Sent Done false
168612024-02-04T14:07:35.413Z 19 Acked 1018 Read 1 Done Sent Done false
168622024-02-04T14:07:35.413Z 20 Acked 1019 Read 7 Done Sent Done false
168632024-02-04T14:07:35.413Z 21 Acked 1020 Read 1 Done Sent Done false
168642024-02-04T14:07:35.413Z 22 Acked 1021 Read 8 Done Sent Done false
168652024-02-04T14:07:35.413Z 23 Acked 1022 Flush 0 Done Sent Done false
168662024-02-04T14:07:35.414Z 24 Acked 1023 Write 4 Done Sent Done false
168672024-02-04T14:07:35.414Z 25 Acked 1024 Read 3 Done Sent Done false
168682024-02-04T14:07:35.414Z 26 Acked 1025 Flush 0 Done Sent Done false
168692024-02-04T14:07:35.414Z 27 Acked 1026 Write 6 Done Sent Done false
168702024-02-04T14:07:35.414Z 28 Acked 1027 Read 10 Done Sent Done false
168712024-02-04T14:07:35.414Z 29 Acked 1028 Read 8 Done Sent Done false
168722024-02-04T14:07:35.415Z 30 Acked 1029 Read 7 Done Sent Done false
168732024-02-04T14:07:35.415Z STATES DS:0 DS:1 DS:2 TOTAL
168742024-02-04T14:07:35.415Z New 0 0 0 0
168752024-02-04T14:07:35.415Z Sent 0 22 0 22
168762024-02-04T14:07:35.415Z Done 30 8 30 68
168772024-02-04T14:07:35.415Z Skipped 0 0 0 0
168782024-02-04T14:07:35.415Z Error 0 0 0 0
168792024-02-04T14:07:35.416ZLast Flush: 1025 0 1025
168802024-02-04T14:07:35.416ZDownstairs last five completed:
168812024-02-04T14:07:35.416ZUpstairs last five completed: 30 29 28 27 26
168822024-02-04T14:07:35.416Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:34.420901781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1753}
168832024-02-04T14:07:35.416ZWrote out file "/tmp/test_repair_verify.data"
168842024-02-04T14:07:35.446Z
168852024-02-04T14:07:35.446ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168862024-02-04T14:07:35.465ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168872024-02-04T14:07:35.465Z 0 000-019 147 147 147 252 252 252 F F F
168882024-02-04T14:07:35.465Z 1 020-039 154 154 154 267 267 267 F F F
168892024-02-04T14:07:35.465Z 2 040-059 152 152 152 264 264 264 F F F
168902024-02-04T14:07:35.466Z 3 060-079 152 152 152 264 264 264 F F F
168912024-02-04T14:07:35.466Z 4 080-099 146 146 146 251 251 251 F F F
168922024-02-04T14:07:35.466Z 5 100-119 152 152 152 262 262 262 F F F
168932024-02-04T14:07:35.466Z 6 120-139 152 152 152 265 265 265 F F F
168942024-02-04T14:07:35.466Z 7 140-159 156 153 156 269 266 269 F F F <---
168952024-02-04T14:07:35.466Z 8 160-179 156 154 156 271 267 271 F F F <---
168962024-02-04T14:07:35.466Z 9 180-199 154 154 154 267 267 267 F F F
168972024-02-04T14:07:35.467Z 10 200-219 154 154 154 267 267 267 F F F
168982024-02-04T14:07:35.467Z 11 220-239 156 154 156 268 267 268 T T T <---
168992024-02-04T14:07:35.467Z 12 240-259 156 153 156 270 266 270 T F T <---
169002024-02-04T14:07:35.467Z 13 260-279 153 153 153 266 266 266 F F F
169012024-02-04T14:07:35.467Z 14 280-299 152 152 152 263 263 263 F F F
169022024-02-04T14:07:35.467Z 15 300-319 156 150 156 268 260 268 F T F <---
169032024-02-04T14:07:35.468Z 16 320-339 156 150 156 269 259 269 F F F <---
169042024-02-04T14:07:35.468Z 17 340-359 156 149 156 269 258 269 F F F <---
169052024-02-04T14:07:35.468Z 18 360-379 156 152 156 269 265 269 F F F <---
169062024-02-04T14:07:35.468Z 19 380-399 154 154 154 267 267 267 F F F
169072024-02-04T14:07:35.468Z 20 400-419 148 148 148 257 257 257 F F F
169082024-02-04T14:07:35.468Z 21 420-439 149 149 149 258 258 258 F F F
169092024-02-04T14:07:35.468Z 22 440-459 152 152 152 263 263 263 F F F
169102024-02-04T14:07:35.469Z 23 460-479 154 154 154 267 267 267 F F F
169112024-02-04T14:07:35.469Z 24 480-499 154 154 154 267 267 267 F F F
169122024-02-04T14:07:35.469Z 25 500-519 152 152 152 263 263 263 F F F
169132024-02-04T14:07:35.469Z 26 520-539 152 152 152 262 262 262 F F F
169142024-02-04T14:07:35.469Z 27 540-559 148 148 148 256 256 256 F F F
169152024-02-04T14:07:35.469Z 28 560-579 150 150 150 260 260 260 F F F
169162024-02-04T14:07:35.470Z 29 580-599 124 124 124 205 205 205 F F F
169172024-02-04T14:07:35.470ZMax gen: 156, Max flush: 271
169182024-02-04T14:07:35.470ZError: Difference in extent metadata found!
169192024-02-04T14:07:35.470ZA Difference in extent metadata is expected here
169202024-02-04T14:07:35.470ZOn loop 78
169212024-02-04T14:07:35.470Z
169222024-02-04T14:07:35.470ZVerifying data now
169232024-02-04T14:07:41.198ZNo change after verify
169242024-02-04T14:07:41.199ZLoop: 78 Downstairs dump after verify (and repair):
169252024-02-04T14:07:41.223ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169262024-02-04T14:07:41.223Z 0 000-019 147 147 147 252 252 252 F F F
169272024-02-04T14:07:41.223Z 1 020-039 154 154 154 267 267 267 F F F
169282024-02-04T14:07:41.223Z 2 040-059 152 152 152 264 264 264 F F F
169292024-02-04T14:07:41.223Z 3 060-079 152 152 152 264 264 264 F F F
169302024-02-04T14:07:41.223Z 4 080-099 146 146 146 251 251 251 F F F
169312024-02-04T14:07:41.223Z 5 100-119 152 152 152 262 262 262 F F F
169322024-02-04T14:07:41.223Z 6 120-139 152 152 152 265 265 265 F F F
169332024-02-04T14:07:41.224Z 7 140-159 156 156 156 269 269 269 F F F
169342024-02-04T14:07:41.224Z 8 160-179 156 156 156 271 271 271 F F F
169352024-02-04T14:07:41.224Z 9 180-199 154 154 154 267 267 267 F F F
169362024-02-04T14:07:41.224Z 10 200-219 154 154 154 267 267 267 F F F
169372024-02-04T14:07:41.224Z 11 220-239 157 157 157 272 272 272 F F F
169382024-02-04T14:07:41.247Z 12 240-259 157 157 157 272 272 272 F F F
169392024-02-04T14:07:41.247Z 13 260-279 153 153 153 266 266 266 F F F
169402024-02-04T14:07:41.248Z 14 280-299 152 152 152 263 263 263 F F F
169412024-02-04T14:07:41.248Z 15 300-319 156 156 156 268 268 268 F F F
169422024-02-04T14:07:41.248Z 16 320-339 156 156 156 269 269 269 F F F
169432024-02-04T14:07:41.248Z 17 340-359 156 156 156 269 269 269 F F F
169442024-02-04T14:07:41.248Z 18 360-379 156 156 156 269 269 269 F F F
169452024-02-04T14:07:41.248Z 19 380-399 154 154 154 267 267 267 F F F
169462024-02-04T14:07:41.248Z 20 400-419 148 148 148 257 257 257 F F F
169472024-02-04T14:07:41.248Z 21 420-439 149 149 149 258 258 258 F F F
169482024-02-04T14:07:41.248Z 22 440-459 152 152 152 263 263 263 F F F
169492024-02-04T14:07:41.248Z 23 460-479 154 154 154 267 267 267 F F F
169502024-02-04T14:07:41.248Z 24 480-499 154 154 154 267 267 267 F F F
169512024-02-04T14:07:41.248Z 25 500-519 152 152 152 263 263 263 F F F
169522024-02-04T14:07:41.248Z 26 520-539 152 152 152 262 262 262 F F F
169532024-02-04T14:07:41.248Z 27 540-559 148 148 148 256 256 256 F F F
169542024-02-04T14:07:41.248Z 28 560-579 150 150 150 260 260 260 F F F
169552024-02-04T14:07:41.248Z 29 580-599 124 124 124 205 205 205 F F F
169562024-02-04T14:07:41.248ZMax gen: 157, Max flush: 272
169572024-02-04T14:07:41.248Z
169582024-02-04T14:07:41.249ZBegin loop 79
169592024-02-04T14:07:41.249ZDownstairs to restart: 1
169602024-02-04T14:07:41.266Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.288286755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761}
169612024-02-04T14:07:41.267Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.288752502Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761}
169622024-02-04T14:07:41.311Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
169632024-02-04T14:07:41.311Z,"time":"2024-02-04T14:07:40.288827235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761}
169642024-02-04T14:07:41.312Z{"msg":"Crucible 13e886b5-3b12-4f0f-a19e-0f4453afe7cc has session id: bbcc2d27-2e47-45e6-96d7-828efb35a8ce","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289036033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169652024-02-04T14:07:41.312Z{"msg":"Upstairs opts: Upstairs UUID: 13e886b5-3b12-4f0f-a19e-0f4453afe7cc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289091547Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169662024-02-04T14:07:41.312Z{"msg":"Crucible stats registered with UUID: 13e886b5-3b12-4f0f-a19e-0f4453afe7cc","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289145735Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169672024-02-04T14:07:41.312Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289175187Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761}
169682024-02-04T14:07:41.312Z{"msg":"13e886b5-3b12-4f0f-a19e-0f4453afe7cc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.28920883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169692024-02-04T14:07:41.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289281387Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169702024-02-04T14:07:41.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289377374Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169712024-02-04T14:07:41.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289419417Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169722024-02-04T14:07:41.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289444755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169732024-02-04T14:07:41.312Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289482742Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169742024-02-04T14:07:41.312Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289523207Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169752024-02-04T14:07:41.313Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289561924Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"io task","client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169762024-02-04T14:07:41.313Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289639676Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"io task","client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169772024-02-04T14:07:41.313Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289685391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"io task","client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169782024-02-04T14:07:41.313Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.289786268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"io task","client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169792024-02-04T14:07:41.313Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.290047967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"io task","client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169802024-02-04T14:07:41.313Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.290089695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"io task","client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169812024-02-04T14:07:41.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.290802382Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169822024-02-04T14:07:41.313Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.290885172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169832024-02-04T14:07:41.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.290930256Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169842024-02-04T14:07:41.313Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.290969642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169852024-02-04T14:07:41.313Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.291714091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169862024-02-04T14:07:41.313Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.292071016Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169872024-02-04T14:07:41.314Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.292513168Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169882024-02-04T14:07:41.314Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.292696317Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169892024-02-04T14:07:41.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.292731729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169902024-02-04T14:07:41.314Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.292828075Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169912024-02-04T14:07:41.314Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.292908213Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169922024-02-04T14:07:41.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.292936197Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169932024-02-04T14:07:41.314Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.292979508Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169942024-02-04T14:07:41.314Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.293004183Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169952024-02-04T14:07:41.314Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.293613289Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169962024-02-04T14:07:41.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294295071Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169972024-02-04T14:07:41.314Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.29436915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169982024-02-04T14:07:41.314Z{"msg":"[0]R flush_numbers[0..12]: [252, 267, 264, 264, 251, 262, 265, 269, 271, 267, 267, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.29441334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
169992024-02-04T14:07:41.315Z{"msg":"[0]R generation[0..12]: [147, 154, 152, 152, 146, 152, 152, 156, 156, 154, 154, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294461803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170002024-02-04T14:07:41.315Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294487761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170012024-02-04T14:07:41.315Z{"msg":"[1]R flush_numbers[0..12]: [252, 267, 264, 264, 251, 262, 265, 269, 271, 267, 267, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294511548Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170022024-02-04T14:07:41.315Z{"msg":"[1]R generation[0..12]: [147, 154, 152, 152, 146, 152, 152, 156, 156, 154, 154, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294534867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170032024-02-04T14:07:41.315Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
170042024-02-04T14:07:41.315Z2024-02-04T14:07:40.294576634Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170052024-02-04T14:07:41.315Z{"msg":"[2]R flush_numbers[0..12]: [252, 267, 264, 264, 251, 262, 265, 269, 271, 267, 267, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294605884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170062024-02-04T14:07:41.315Z{"msg":"[2]R generation[0..12]: [147, 154, 152, 152, 146, 152, 152, 156, 156, 154, 154, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294629078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170072024-02-04T14:07:41.315Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294671764Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170082024-02-04T14:07:41.315Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294694837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170092024-02-04T14:07:41.315Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294735588Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170102024-02-04T14:07:41.315Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294757966Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170112024-02-04T14:07:41.315Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294779673Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"":"downstairs","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170122024-02-04T14:07:41.316Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294826282Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170132024-02-04T14:07:41.316Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294847338Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"0","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170142024-02-04T14:07:41.316Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294869173Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"1","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170152024-02-04T14:07:41.316Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294915153Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"client":"2","":"downstairs","session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170162024-02-04T14:07:41.316Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294938437Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170172024-02-04T14:07:41.316Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294959391Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170182024-02-04T14:07:41.316Z{"msg":"13e886b5-3b12-4f0f-a19e-0f4453afe7cc is now active with session: bbcc2d27-2e47-45e6-96d7-828efb35a8ce","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.294981027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170192024-02-04T14:07:41.316Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.295000883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761,"session_id":"bbcc2d27-2e47-45e6-96d7-828efb35a8ce"}
170202024-02-04T14:07:41.316Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:40.295021335Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761}
170212024-02-04T14:07:41.316ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170222024-02-04T14:07:41.316ZLoading write count information from file "/tmp/test_repair_verify.data"
170232024-02-04T14:07:41.316ZRun Repair workload
170242024-02-04T14:07:41.317Z01/30 Write block 3 len 20480 data: 6 9 10 11 11
170252024-02-04T14:07:41.317Z02/30 Read block 93 len 4096
170262024-02-04T14:07:41.317Z03/30 Write block 206 len 16384 data: 14 13 11 10
170272024-02-04T14:07:41.317Z04/30 Read block 445 len 28672
170282024-02-04T14:07:41.317Z05/30 Flush
170292024-02-04T14:07:41.351Z06/30 Flush
170302024-02-04T14:07:41.351Z07/30 Write block 226 len 8192 data: 10 10
170312024-02-04T14:07:41.351Z08/30 Read block 408 len 12288
170322024-02-04T14:07:41.351Z09/30 Write block 457 len 40960 data: 16 16 16 15 14 11 12 12 15 17
170332024-02-04T14:07:41.351Z10/30 Read block 127 len 8192
170342024-02-04T14:07:41.351Z11/30 Write block 92 len 12288 data: 7 6 7
170352024-02-04T14:07:41.351Z12/30 Write block 47 len 36864 data: 9 11 8 7 10 11 11 10 10
170362024-02-04T14:07:41.351Z13/30 Read block 311 len 4096
170372024-02-04T14:07:41.352Z14/30 Write block 431 len 28672 data: 11 11 11 15 15 14 12
170382024-02-04T14:07:41.352Z15/30 Read block 344 len 24576
170392024-02-04T14:07:41.352Z16/30 Flush
170402024-02-04T14:07:41.352Z17/30 Write block 529 len 20480 data: 14 12 9 9 7
170412024-02-04T14:07:41.352Z18/30 Read block 327 len 40960
170422024-02-04T14:07:41.352Z19/30 Flush
170432024-02-04T14:07:41.352Z20/30 Write block 568 len 4096 data: 14
170442024-02-04T14:07:41.352Z21/30 Read block 300 len 8192
170452024-02-04T14:07:41.352Z22/30 Write block 536 len 36864 data: 7 5 8 10 12 12 11 14 12
170462024-02-04T14:07:41.352Z23/30 Write block 408 len 8192 data: 13 14
170472024-02-04T14:07:41.353Z24/30 Read block 82 len 20480
170482024-02-04T14:07:41.353Z25/30 Write block 545 len 28672 data: 11 8 8 8 9 9 7
170492024-02-04T14:07:41.353Z26/30 Read block 56 len 16384
170502024-02-04T14:07:41.353Z27/30 Write block 193 len 24576 data: 11 10 9 8 10 12
170512024-02-04T14:07:41.353Z28/30 Write block 572 len 16384 data: 11 10 10 11
170522024-02-04T14:07:41.353Z29/30 Read block 567 len 36864
170532024-02-04T14:07:41.353Z30/30 Read block 202 len 36864
170542024-02-04T14:07:41.353Z----------------------------------------------------------------
170552024-02-04T14:07:41.353Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
170562024-02-04T14:07:41.353ZStates: Active Active Active
170572024-02-04T14:07:41.353ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170582024-02-04T14:07:41.354Z 1 Acked 1000 Write 5 Done Sent Done false
170592024-02-04T14:07:41.354Z 2 Acked 1001 Read 1 Done Sent Done false
170602024-02-04T14:07:41.354Z 3 Acked 1002 Write 4 Done Sent Done false
170612024-02-04T14:07:41.354Z 4 Acked 1003 Read 7 Done Sent Done false
170622024-02-04T14:07:41.354Z 5 Acked 1004 Flush 0 Done Sent Done false
170632024-02-04T14:07:41.354Z 6 Acked 1005 Flush 0 Done Sent Done false
170642024-02-04T14:07:41.354Z 7 Acked 1006 Write 2 Done Sent Done false
170652024-02-04T14:07:41.354Z 8 Acked 1007 Read 3 Done Sent Done false
170662024-02-04T14:07:41.354Z 9 Acked 1008 Write 10 Done Sent Done false
170672024-02-04T14:07:41.354Z 10 Acked 1009 Read 2 Done Sent Done false
170682024-02-04T14:07:41.354Z 11 Acked 1010 Write 3 Done Sent Done false
170692024-02-04T14:07:41.354Z 12 Acked 1011 Write 9 Done Sent Done false
170702024-02-04T14:07:41.354Z 13 Acked 1012 Read 1 Done Sent Done false
170712024-02-04T14:07:41.355Z 14 Acked 1013 Write 7 Done Sent Done false
170722024-02-04T14:07:41.355Z 15 Acked 1014 Read 6 Done Sent Done false
170732024-02-04T14:07:41.355Z 16 Acked 1015 Flush 0 Done Sent Done false
170742024-02-04T14:07:41.355Z 17 Acked 1016 Write 5 Done Sent Done false
170752024-02-04T14:07:41.355Z 18 Acked 1017 Read 10 Done Sent Done false
170762024-02-04T14:07:41.355Z 19 Acked 1018 Flush 0 Done Sent Done false
170772024-02-04T14:07:41.355Z 20 Acked 1019 Write 1 Done Sent Done false
170782024-02-04T14:07:41.355Z 21 Acked 1020 Read 2 Done Sent Done false
170792024-02-04T14:07:41.355Z 22 Acked 1021 Write 9 Done Sent Done false
170802024-02-04T14:07:41.355Z 23 Acked 1022 Write 2 Done Sent Done false
170812024-02-04T14:07:41.355Z 24 Acked 1023 Read 5 Done Sent Done false
170822024-02-04T14:07:41.355Z 25 Acked 1024 Write 7 Done Sent Done false
170832024-02-04T14:07:41.355Z 26 Acked 1025 Read 4 Done Sent Done false
170842024-02-04T14:07:41.356Z 27 Acked 1026 Write 6 Done Sent Done false
170852024-02-04T14:07:41.356Z 28 Acked 1027 Write 4 Done Sent Done false
170862024-02-04T14:07:41.356Z 29 Acked 1028 Read 9 Done Sent Done false
170872024-02-04T14:07:41.356Z 30 Acked 1029 Read 9 Done Sent Sent false
170882024-02-04T14:07:41.356Z STATES DS:0 DS:1 DS:2 TOTAL
170892024-02-04T14:07:41.356Z New 0 0 0 0
170902024-02-04T14:07:41.356Z Sent 0 30 1 31
170912024-02-04T14:07:41.356Z Done 30 0 29 59
170922024-02-04T14:07:41.356Z Skipped 0 0 0 0
170932024-02-04T14:07:41.356Z Error 0 0 0 0
170942024-02-04T14:07:41.356ZLast Flush: 1018 0 1018
170952024-02-04T14:07:41.356ZDownstairs last five completed:
170962024-02-04T14:07:41.356ZUpstairs last five completed: 30 29 28 27 26
170972024-02-04T14:07:41.357Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:40.342776196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1761}
170982024-02-04T14:07:41.357ZWrote out file "/tmp/test_repair_verify.data"
170992024-02-04T14:07:41.357Z
171002024-02-04T14:07:41.357ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171012024-02-04T14:07:41.777ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171022024-02-04T14:07:41.777Z 0 000-019 158 147 158 273 252 273 F F F <---
171032024-02-04T14:07:41.778Z 1 020-039 154 154 154 267 267 267 F F F
171042024-02-04T14:07:41.778Z 2 040-059 158 152 158 275 264 275 F F F <---
171052024-02-04T14:07:41.778Z 3 060-079 152 152 152 264 264 264 F F F
171062024-02-04T14:07:41.778Z 4 080-099 158 146 158 275 251 275 F F F <---
171072024-02-04T14:07:41.778Z 5 100-119 152 152 152 262 262 262 F F F
171082024-02-04T14:07:41.778Z 6 120-139 152 152 152 265 265 265 F F F
171092024-02-04T14:07:41.778Z 7 140-159 156 156 156 269 269 269 F F F
171102024-02-04T14:07:41.778Z 8 160-179 156 156 156 271 271 271 F F F
171112024-02-04T14:07:41.778Z 9 180-199 154 154 154 267 267 267 T F T <---
171122024-02-04T14:07:41.778Z 10 200-219 158 154 158 273 267 273 F F F <---
171132024-02-04T14:07:41.778Z 11 220-239 158 157 158 275 272 275 F F F <---
171142024-02-04T14:07:41.778Z 12 240-259 157 157 157 272 272 272 F F F
171152024-02-04T14:07:41.778Z 13 260-279 153 153 153 266 266 266 F F F
171162024-02-04T14:07:41.778Z 14 280-299 152 152 152 263 263 263 F F F
171172024-02-04T14:07:41.778Z 15 300-319 156 156 156 268 268 268 F F F
171182024-02-04T14:07:41.778Z 16 320-339 156 156 156 269 269 269 F F F
171192024-02-04T14:07:41.778Z 17 340-359 156 156 156 269 269 269 F F F
171202024-02-04T14:07:41.778Z 18 360-379 156 156 156 269 269 269 F F F
171212024-02-04T14:07:41.778Z 19 380-399 154 154 154 267 267 267 F F F
171222024-02-04T14:07:41.778Z 20 400-419 148 148 148 257 257 257 T F T <---
171232024-02-04T14:07:41.778Z 21 420-439 158 149 158 275 258 275 F F F <---
171242024-02-04T14:07:41.778Z 22 440-459 158 152 158 275 263 275 F F F <---
171252024-02-04T14:07:41.778Z 23 460-479 158 154 158 275 267 275 F F F <---
171262024-02-04T14:07:41.778Z 24 480-499 154 154 154 267 267 267 F F F
171272024-02-04T14:07:41.779Z 25 500-519 152 152 152 263 263 263 F F F
171282024-02-04T14:07:41.779Z 26 520-539 158 152 158 276 262 276 T F T <---
171292024-02-04T14:07:41.779Z 27 540-559 148 148 148 256 256 256 T F T <---
171302024-02-04T14:07:41.779Z 28 560-579 150 150 150 260 260 260 T F T <---
171312024-02-04T14:07:41.779Z 29 580-599 124 124 124 205 205 205 F F F
171322024-02-04T14:07:41.779ZMax gen: 158, Max flush: 276
171332024-02-04T14:07:41.779ZError: Difference in extent metadata found!
171342024-02-04T14:07:41.779ZA Difference in extent metadata is expected here
171352024-02-04T14:07:41.779ZOn loop 79
171362024-02-04T14:07:41.779Z
171372024-02-04T14:07:41.779ZVerifying data now
171382024-02-04T14:07:46.478ZNo change after verify
171392024-02-04T14:07:46.478ZLoop: 79 Downstairs dump after verify (and repair):
171402024-02-04T14:07:46.500ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171412024-02-04T14:07:46.500Z 0 000-019 158 158 158 273 273 273 F F F
171422024-02-04T14:07:46.500Z 1 020-039 154 154 154 267 267 267 F F F
171432024-02-04T14:07:46.501Z 2 040-059 158 158 158 275 275 275 F F F
171442024-02-04T14:07:46.501Z 3 060-079 152 152 152 264 264 264 F F F
171452024-02-04T14:07:46.501Z 4 080-099 158 158 158 275 275 275 F F F
171462024-02-04T14:07:46.501Z 5 100-119 152 152 152 262 262 262 F F F
171472024-02-04T14:07:46.501Z 6 120-139 152 152 152 265 265 265 F F F
171482024-02-04T14:07:46.501Z 7 140-159 156 156 156 269 269 269 F F F
171492024-02-04T14:07:46.502Z 8 160-179 156 156 156 271 271 271 F F F
171502024-02-04T14:07:46.502Z 9 180-199 159 159 159 277 277 277 F F F
171512024-02-04T14:07:46.508Z 10 200-219 158 158 158 273 273 273 F F F
171522024-02-04T14:07:46.508Z 11 220-239 158 158 158 275 275 275 F F F
171532024-02-04T14:07:46.508Z 12 240-259 157 157 157 272 272 272 F F F
171542024-02-04T14:07:46.509Z 13 260-279 153 153 153 266 266 266 F F F
171552024-02-04T14:07:46.509Z 14 280-299 152 152 152 263 263 263 F F F
171562024-02-04T14:07:46.509Z 15 300-319 156 156 156 268 268 268 F F F
171572024-02-04T14:07:46.509Z 16 320-339 156 156 156 269 269 269 F F F
171582024-02-04T14:07:46.509Z 17 340-359 156 156 156 269 269 269 F F F
171592024-02-04T14:07:46.509Z 18 360-379 156 156 156 269 269 269 F F F
171602024-02-04T14:07:46.509Z 19 380-399 154 154 154 267 267 267 F F F
171612024-02-04T14:07:46.509Z 20 400-419 159 159 159 277 277 277 F F F
171622024-02-04T14:07:46.509Z 21 420-439 158 158 158 275 275 275 F F F
171632024-02-04T14:07:46.509Z 22 440-459 158 158 158 275 275 275 F F F
171642024-02-04T14:07:46.509Z 23 460-479 158 158 158 275 275 275 F F F
171652024-02-04T14:07:46.509Z 24 480-499 154 154 154 267 267 267 F F F
171662024-02-04T14:07:46.509Z 25 500-519 152 152 152 263 263 263 F F F
171672024-02-04T14:07:46.509Z 26 520-539 159 159 159 277 277 277 F F F
171682024-02-04T14:07:46.509Z 27 540-559 159 159 159 277 277 277 F F F
171692024-02-04T14:07:46.510Z 28 560-579 159 159 159 277 277 277 F F F
171702024-02-04T14:07:46.510Z 29 580-599 124 124 124 205 205 205 F F F
171712024-02-04T14:07:46.510ZMax gen: 159, Max flush: 277
171722024-02-04T14:07:46.510Z
171732024-02-04T14:07:46.510ZBegin loop 80
171742024-02-04T14:07:46.510ZDownstairs to restart: 2
171752024-02-04T14:07:46.542Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.567590453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769}
171762024-02-04T14:07:46.542Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568019175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769}
171772024-02-04T14:07:46.542Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568047941Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769}
171782024-02-04T14:07:46.542Z{"msg":"Crucible 0d4d627a-c061-4592-a7c8-d3563f336a42 has session id: 95ae3479-4b3d-4c9f-9c29-f3d0063f9923","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568078062Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171792024-02-04T14:07:46.558Z{"msg":"Upstairs opts: Upstairs UUID: 0d4d627a-c061-4592-a7c8-d3563f336a42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568130835Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171802024-02-04T14:07:46.558Z{"msg":"Crucible stats registered with UUID: 0d4d627a-c061-4592-a7c8-d3563f336a42","v":0,"name":"crucible","level":30Crucible runtime is spawned
171812024-02-04T14:07:46.558Z,"time":"2024-02-04T14:07:45.568254806Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171822024-02-04T14:07:46.559Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568430827Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769}
171832024-02-04T14:07:46.559Z{"msg":"0d4d627a-c061-4592-a7c8-d3563f336a42 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568707564Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171842024-02-04T14:07:46.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568764417Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171852024-02-04T14:07:46.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568789685Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171862024-02-04T14:07:46.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568816729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171872024-02-04T14:07:46.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568857311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171882024-02-04T14:07:46.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568925013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171892024-02-04T14:07:46.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.568970996Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171902024-02-04T14:07:46.559Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.569066947Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"io task","client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171912024-02-04T14:07:46.559Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.569111526Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"io task","client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171922024-02-04T14:07:46.559Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.569191812Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"io task","client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171932024-02-04T14:07:46.559Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.569238534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"io task","client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171942024-02-04T14:07:46.560Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.569311659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"io task","client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171952024-02-04T14:07:46.560Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.56934905Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"io task","client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171962024-02-04T14:07:46.560Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.570615946Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171972024-02-04T14:07:46.560Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.570700653Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171982024-02-04T14:07:46.560Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.57074894Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
171992024-02-04T14:07:46.560Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.570882412Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172002024-02-04T14:07:46.560Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.57175105Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172012024-02-04T14:07:46.560Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.571847929Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172022024-02-04T14:07:46.560Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.572087244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172032024-02-04T14:07:46.561Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.57220207Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172042024-02-04T14:07:46.561Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.572682514Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172052024-02-04T14:07:46.561Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.572751682Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172062024-02-04T14:07:46.561Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.57277908Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172072024-02-04T14:07:46.561Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.572800613Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172082024-02-04T14:07:46.561Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.572823114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172092024-02-04T14:07:46.561Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.572845426Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172102024-02-04T14:07:46.561Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.573502759Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172112024-02-04T14:07:46.562Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574208791Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172122024-02-04T14:07:46.562Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574292466Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172132024-02-04T14:07:46.562Z{"msg":"[0]R flush_numbers[0..12]: [273, 267, 275, 264, 275, 262, 265, 269, 271, 277, 273, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574325547Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172142024-02-04T14:07:46.562Z{"msg":"[0]R generation[0..12]: [158, 154, 158, 152, 158, 152, 152, 156, 156, 159, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574356509Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172152024-02-04T14:07:46.562Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574384754Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172162024-02-04T14:07:46.562Z{"msg":"[1]R flush_numbers[0..12]: [273, 267, 275, 264, 275, 262, 265, 269, 271, 277, 273, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574430061Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172172024-02-04T14:07:46.562Z{"msg":"[1]R generation[0..12]: [158, 154, 158, 152, 158, 152, 152, 156, 156, 159, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574455118Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172182024-02-04T14:07:46.563Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574532094Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172192024-02-04T14:07:46.563Z{"msg":"[2]R flush_numbers[0..12]: [273, 267, 275, 264, 275, 262, 265, 269, 271, 277, 273, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.57455699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"Wait for a query_work_queue command to finish before sending IO
172202024-02-04T14:07:46.563Z}
172212024-02-04T14:07:46.563Z{"msg":"[2]R generation[0..12]: [158, 154, 158, 152, 158, 152, 152, 156, 156, 159, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.57464389Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172222024-02-04T14:07:46.563Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574720099Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172232024-02-04T14:07:46.563Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574744287Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172242024-02-04T14:07:46.563Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574785367Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172252024-02-04T14:07:46.563Z{"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574807944Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172262024-02-04T14:07:46.563Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574857629Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"":"downstairs","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172272024-02-04T14:07:46.563Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574882534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172282024-02-04T14:07:46.563Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.574930031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"0","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172292024-02-04T14:07:46.563Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.57495453Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"1","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172302024-02-04T14:07:46.564Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.575020043Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"client":"2","":"downstairs","session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172312024-02-04T14:07:46.564Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.575045585Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172322024-02-04T14:07:46.564Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.575100134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172332024-02-04T14:07:46.564Z{"msg":"0d4d627a-c061-4592-a7c8-d3563f336a42 is now active with session: 95ae3479-4b3d-4c9f-9c29-f3d0063f9923","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.575124387Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172342024-02-04T14:07:46.564Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.575153097Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769,"session_id":"95ae3479-4b3d-4c9f-9c29-f3d0063f9923"}
172352024-02-04T14:07:46.564Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:45.575174873Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769}
172362024-02-04T14:07:46.564ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172372024-02-04T14:07:46.564ZLoading write count information from file "/tmp/test_repair_verify.data"
172382024-02-04T14:07:46.564ZRun Repair workload
172392024-02-04T14:07:46.564Z01/30 Flush
172402024-02-04T14:07:46.564Z02/30 Write block 2 len 40960 data: 6 7 10 11 12 12 10 12 8 7
172412024-02-04T14:07:46.564Z03/30 Read block 443 len 32768
172422024-02-04T14:07:46.570Z04/30 Write block 378 len 8192 data: 9 9
172432024-02-04T14:07:46.570Z05/30 Read block 73 len 24576
172442024-02-04T14:07:46.570Z06/30 Read block 366 len 40960
172452024-02-04T14:07:46.571Z07/30 Write block 499 len 12288 data: 16 16 16
172462024-02-04T14:07:46.571Z08/30 Flush
172472024-02-04T14:07:46.571Z09/30 Write block 485 len 4096 data: 12
172482024-02-04T14:07:46.571Z10/30 Read block 520 len 24576
172492024-02-04T14:07:46.571Z11/30 Write block 265 len 24576 data: 12 13 10 13 12 12
172502024-02-04T14:07:46.571Z12/30 Write block 375 len 4096 data: 12
172512024-02-04T14:07:46.572Z13/30 Write block 262 len 40960 data: 14 14 12 13 14 11 14 13 13 11
172522024-02-04T14:07:46.572Z14/30 Read block 576 len 12288
172532024-02-04T14:07:46.572Z15/30 Read block 338 len 8192
172542024-02-04T14:07:46.572Z16/30 Write block 137 len 36864 data: 7 8 10 15 11 7 8 11 12
172552024-02-04T14:07:46.572Z17/30 Read block 508 len 12288
172562024-02-04T14:07:46.581Z18/30 Write block 305 len 40960 data: 13 12 12 11 11 13 12 9 10 10
172572024-02-04T14:07:46.581Z19/30 Flush
172582024-02-04T14:07:46.581Z20/30 Write block 436 len 20480 data: 15 13 10 8 11
172592024-02-04T14:07:46.581Z21/30 Read block 307 len 16384
172602024-02-04T14:07:46.589Z22/30 Write block 42 len 8192 data: 12 10
172612024-02-04T14:07:46.590Z23/30 Write block 326 len 36864 data: 9 8 7 7 8 9 9 10 13
172622024-02-04T14:07:46.590Z24/30 Read block 540 len 28672
172632024-02-04T14:07:46.590Z25/30 Read block 439 len 12288
172642024-02-04T14:07:46.590Z26/30 Read block 561 len 4096
172652024-02-04T14:07:46.591Z27/30 Read block 326 len 4096
172662024-02-04T14:07:46.600Z28/30 Write block 86 len 12288 data: 13 9 8
172672024-02-04T14:07:46.600Z29/30 Read block 518 len 36864
172682024-02-04T14:07:46.600Z30/30 Read block 173 len 36864
172692024-02-04T14:07:46.601Z----------------------------------------------------------------
172702024-02-04T14:07:46.601Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:29
172712024-02-04T14:07:46.601ZStates: Active Active Active
172722024-02-04T14:07:46.601ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172732024-02-04T14:07:46.601Z 2 Acked 1001 Write 10 Done Done Sent false
172742024-02-04T14:07:46.602Z 3 Acked 1002 Read 8 Done Done Sent false
172752024-02-04T14:07:46.602Z 4 Acked 1003 Write 2 Done Done Sent false
172762024-02-04T14:07:46.602Z 5 Acked 1004 Read 6 Done Done Sent false
172772024-02-04T14:07:46.602Z 6 Acked 1005 Read 10 Done Done Sent false
172782024-02-04T14:07:46.603Z 7 Acked 1006 Write 3 Done Done Sent false
172792024-02-04T14:07:46.603Z 8 Acked 1007 Flush 0 Done Done Sent false
172802024-02-04T14:07:46.603Z 9 Acked 1008 Write 1 Done Done Sent false
172812024-02-04T14:07:46.603Z 10 Acked 1009 Read 6 Done Done Sent false
172822024-02-04T14:07:46.604Z 11 Acked 1010 Write 6 Done Done Sent false
172832024-02-04T14:07:46.604Z 12 Acked 1011 Write 1 Done Done Sent false
172842024-02-04T14:07:46.604Z 13 Acked 1012 Write 10 Done Done Sent false
172852024-02-04T14:07:46.604Z 14 Acked 1013 Read 3 Done Done Sent false
172862024-02-04T14:07:46.604Z 15 Acked 1014 Read 2 Done Done Sent false
172872024-02-04T14:07:46.605Z 16 Acked 1015 Write 9 Done Done Sent false
172882024-02-04T14:07:46.605Z 17 Acked 1016 Read 3 Done Done Sent false
172892024-02-04T14:07:46.605Z 18 Acked 1017 Write 10 Done Done Sent false
172902024-02-04T14:07:46.605Z 19 Acked 1018 Flush 0 Done Done Sent false
172912024-02-04T14:07:46.605Z 20 Acked 1019 Write 5 Done Done Sent false
172922024-02-04T14:07:46.606Z 21 Acked 1020 Read 4 Done Done Sent false
172932024-02-04T14:07:46.606Z 22 Acked 1021 Write 2 Done Done Sent false
172942024-02-04T14:07:46.606Z 23 Acked 1022 Write 9 Done Done Sent false
172952024-02-04T14:07:46.606Z 24 Acked 1023 Read 7 Done Done Sent false
172962024-02-04T14:07:46.607Z 25 Acked 1024 Read 3 Done Done Sent false
172972024-02-04T14:07:46.607Z 26 Acked 1025 Read 1 Done Done Sent false
172982024-02-04T14:07:46.607Z 27 Acked 1026 Read 1 Done Done Sent false
172992024-02-04T14:07:46.607Z 28 Acked 1027 Write 3 Done Done Sent false
173002024-02-04T14:07:46.608Z 29 Acked 1028 Read 9 Done Done Sent false
173012024-02-04T14:07:46.608Z 30 Acked 1029 Read 9 Sent Done Sent false
173022024-02-04T14:07:46.608Z STATES DS:0 DS:1 DS:2 TOTAL
173032024-02-04T14:07:46.608Z New 0 0 0 0
173042024-02-04T14:07:46.608Z Sent 1 0 29 30
173052024-02-04T14:07:46.609Z Done 28 29 0 57
173062024-02-04T14:07:46.609Z Skipped 0 0 0 0
173072024-02-04T14:07:46.609Z Error 0 0 0 0
173082024-02-04T14:07:46.609ZLast Flush: 1018 1018 1000
173092024-02-04T14:07:46.609ZDownstairs last five completed: 1000
173102024-02-04T14:07:46.610ZUpstairs last five completed: 30 29 28 27 26
173112024-02-04T14:07:46.610Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:45.623977111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1769}
173122024-02-04T14:07:46.610ZWrote out file "/tmp/test_repair_verify.data"
173132024-02-04T14:07:46.619Z
173142024-02-04T14:07:46.619ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173152024-02-04T14:07:46.643ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173162024-02-04T14:07:46.644Z 0 000-019 160 160 158 279 279 273 F F F <---
173172024-02-04T14:07:46.644Z 1 020-039 154 154 154 267 267 267 F F F
173182024-02-04T14:07:46.644Z 2 040-059 158 158 158 275 275 275 T T F <---
173192024-02-04T14:07:46.644Z 3 060-079 152 152 152 264 264 264 F F F
173202024-02-04T14:07:46.654Z 4 080-099 158 158 158 275 275 275 T T F <---
173212024-02-04T14:07:46.655Z 5 100-119 152 152 152 262 262 262 F F F
173222024-02-04T14:07:46.655Z 6 120-139 160 160 152 280 280 265 F F F <---
173232024-02-04T14:07:46.655Z 7 140-159 160 160 156 280 280 269 F F F <---
173242024-02-04T14:07:46.655Z 8 160-179 156 156 156 271 271 271 F F F
173252024-02-04T14:07:46.655Z 9 180-199 159 159 159 277 277 277 F F F
173262024-02-04T14:07:46.655Z 10 200-219 158 158 158 273 273 273 F F F
173272024-02-04T14:07:46.656Z 11 220-239 158 158 158 275 275 275 F F F
173282024-02-04T14:07:46.656Z 12 240-259 157 157 157 272 272 272 F F F
173292024-02-04T14:07:46.656Z 13 260-279 160 160 153 280 280 266 F F F <---
173302024-02-04T14:07:46.656Z 14 280-299 152 152 152 263 263 263 F F F
173312024-02-04T14:07:46.656Z 15 300-319 160 160 156 280 280 268 F F F <---
173322024-02-04T14:07:46.656Z 16 320-339 156 156 156 269 269 269 T T F <---
173332024-02-04T14:07:46.656Z 17 340-359 156 156 156 269 269 269 F F F
173342024-02-04T14:07:46.657Z 18 360-379 160 160 156 280 280 269 F F F <---
173352024-02-04T14:07:46.657Z 19 380-399 154 154 154 267 267 267 F F F
173362024-02-04T14:07:46.657Z 20 400-419 159 159 159 277 277 277 F F F
173372024-02-04T14:07:46.657Z 21 420-439 158 158 158 275 275 275 T T F <---
173382024-02-04T14:07:46.657Z 22 440-459 158 158 158 275 275 275 T T F <---
173392024-02-04T14:07:46.657Z 23 460-479 158 158 158 275 275 275 F F F
173402024-02-04T14:07:46.658Z 24 480-499 160 160 154 280 280 267 F F F <---
173412024-02-04T14:07:46.658Z 25 500-519 160 160 152 279 279 263 F F F <---
173422024-02-04T14:07:46.658Z 26 520-539 159 159 159 277 277 277 F F F
173432024-02-04T14:07:46.658Z 27 540-559 159 159 159 277 277 277 F F F
173442024-02-04T14:07:46.658Z 28 560-579 159 159 159 277 277 277 F F F
173452024-02-04T14:07:46.658Z 29 580-599 124 124 124 205 205 205 F F F
173462024-02-04T14:07:46.659ZMax gen: 160, Max flush: 280
173472024-02-04T14:07:46.659ZError: Difference in extent metadata found!
173482024-02-04T14:07:46.659ZA Difference in extent metadata is expected here
173492024-02-04T14:07:46.659ZOn loop 80
173502024-02-04T14:07:46.659Z
173512024-02-04T14:07:46.659ZVerifying data now
173522024-02-04T14:07:51.766ZNo change after verify
173532024-02-04T14:07:51.766ZLoop: 80 Downstairs dump after verify (and repair):
173542024-02-04T14:07:51.789ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173552024-02-04T14:07:51.789Z 0 000-019 160 160 160 279 279 279 F F F
173562024-02-04T14:07:51.790Z 1 020-039 154 154 154 267 267 267 F F F
173572024-02-04T14:07:51.790Z 2 040-059 161 161 161 281 281 281 F F F
173582024-02-04T14:07:51.790Z 3 060-079 152 152 152 264 264 264 F F F
173592024-02-04T14:07:51.790Z 4 080-099 161 161 161 281 281 281 F F F
173602024-02-04T14:07:51.790Z 5 100-119 152 152 152 262 262 262 F F F
173612024-02-04T14:07:51.790Z 6 120-139 160 160 160 280 280 280 F F F
173622024-02-04T14:07:51.790Z 7 140-159 160 160 160 280 280 280 F F F
173632024-02-04T14:07:51.790Z 8 160-179 156 156 156 271 271 271 F F F
173642024-02-04T14:07:51.790Z 9 180-199 159 159 159 277 277 277 F F F
173652024-02-04T14:07:51.790Z 10 200-219 158 158 158 273 273 273 F F F
173662024-02-04T14:07:51.790Z 11 220-239 158 158 158 275 275 275 F F F
173672024-02-04T14:07:51.794Z 12 240-259 157 157 157 272 272 272 F F F
173682024-02-04T14:07:51.794Z 13 260-279 160 160 160 280 280 280 F F F
173692024-02-04T14:07:51.794Z 14 280-299 152 152 152 263 263 263 F F F
173702024-02-04T14:07:51.794Z 15 300-319 160 160 160 280 280 280 F F F
173712024-02-04T14:07:51.794Z 16 320-339 161 161 161 281 281 281 F F F
173722024-02-04T14:07:51.794Z 17 340-359 156 156 156 269 269 269 F F F
173732024-02-04T14:07:51.794Z 18 360-379 160 160 160 280 280 280 F F F
173742024-02-04T14:07:51.794Z 19 380-399 154 154 154 267 267 267 F F F
173752024-02-04T14:07:51.795Z 20 400-419 159 159 159 277 277 277 F F F
173762024-02-04T14:07:51.795Z 21 420-439 161 161 161 281 281 281 F F F
173772024-02-04T14:07:51.795Z 22 440-459 161 161 161 281 281 281 F F F
173782024-02-04T14:07:51.795Z 23 460-479 158 158 158 275 275 275 F F F
173792024-02-04T14:07:51.795Z 24 480-499 160 160 160 280 280 280 F F F
173802024-02-04T14:07:51.795Z 25 500-519 160 160 160 279 279 279 F F F
173812024-02-04T14:07:51.795Z 26 520-539 159 159 159 277 277 277 F F F
173822024-02-04T14:07:51.795Z 27 540-559 159 159 159 277 277 277 F F F
173832024-02-04T14:07:51.795Z 28 560-579 159 159 159 277 277 277 F F F
173842024-02-04T14:07:51.795Z 29 580-599 124 124 124 205 205 205 F F F
173852024-02-04T14:07:51.795ZMax gen: 161, Max flush: 281
173862024-02-04T14:07:51.799Z
173872024-02-04T14:07:51.799ZBegin loop 81
173882024-02-04T14:07:51.799ZDownstairs to restart: 0
173892024-02-04T14:07:51.828Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.860070108Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777}
173902024-02-04T14:07:51.828Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.860620026Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777}
173912024-02-04T14:07:51.828Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.860666044Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777}
173922024-02-04T14:07:51.834Z{"msg":"Crucible 96162f02-794c-44db-bcc7-953a139e4f58 has session id: e64868b6-a187-45e5-8f78-de858fbf62b3","v":0,"name":"crucible","level":30Crucible runtime is spawned
173932024-02-04T14:07:51.834Z,"time":"2024-02-04T14:07:50.860792122Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
173942024-02-04T14:07:51.834Z{"msg":"Upstairs opts: Upstairs UUID: 96162f02-794c-44db-bcc7-953a139e4f58, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861130451Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
173952024-02-04T14:07:51.834Z{"msg":"Crucible stats registered with UUID: 96162f02-794c-44db-bcc7-953a139e4f58","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861217328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
173962024-02-04T14:07:51.834Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861291978Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777}
173972024-02-04T14:07:51.834Z{"msg":"96162f02-794c-44db-bcc7-953a139e4f58 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861329332Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
173982024-02-04T14:07:51.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861398007Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
173992024-02-04T14:07:51.834Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.86143816Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174002024-02-04T14:07:51.834Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861512196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174012024-02-04T14:07:51.835Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861582184Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174022024-02-04T14:07:51.835Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861683054Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174032024-02-04T14:07:51.835Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861724371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174042024-02-04T14:07:51.835Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861801888Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174052024-02-04T14:07:51.835Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861845252Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174062024-02-04T14:07:51.835Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.86191277Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174072024-02-04T14:07:51.835Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.861954523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174082024-02-04T14:07:51.835Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:50.862040135Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174092024-02-04T14:07:51.835Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.862114372Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174102024-02-04T14:07:51.835Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:50.862199842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174112024-02-04T14:07:51.835Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:50.862248528Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174122024-02-04T14:07:51.835Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.862287178Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174132024-02-04T14:07:51.835Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.862360927Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174142024-02-04T14:07:51.836Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.862536231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174152024-02-04T14:07:51.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.863255474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174162024-02-04T14:07:51.836Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.863340023Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174172024-02-04T14:07:51.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.863446253Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174182024-02-04T14:07:51.836Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.863510358Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174192024-02-04T14:07:51.836Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.864257223Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174202024-02-04T14:07:51.836Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.864374725Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174212024-02-04T14:07:51.840Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.865409669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174222024-02-04T14:07:51.840Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.865473304Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174232024-02-04T14:07:51.840Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.865499385Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174242024-02-04T14:07:51.840Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.865540294Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174252024-02-04T14:07:51.840Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.865564289Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174262024-02-04T14:07:51.840Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:50.865605375Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174272024-02-04T14:07:52.832Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.86320425Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174282024-02-04T14:07:52.843Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.863419853Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174292024-02-04T14:07:52.843Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.864367008Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174302024-02-04T14:07:52.843Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.864428666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174312024-02-04T14:07:52.843Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.865246619Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174322024-02-04T14:07:52.843Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.865994475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174332024-02-04T14:07:52.843Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866063746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174342024-02-04T14:07:52.844Z{"msg":"[0]R flush_numbers[0..12]: [279, 267, 281, 264, 281, 262, 280, 280, 271, 277, 273, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866090584Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174352024-02-04T14:07:52.844Z{"msg":"[0]R generation[0..12]: [160, 154, 161, 152, 161, 152, 160, 160, 156, 159, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866189399Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174362024-02-04T14:07:52.844Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866216117Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174372024-02-04T14:07:52.844Z{"msg":"[1]R flush_numbers[0..12]: [279, 267, 281, 264, 281, 262, 280, 280, 271, 277, 273, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866258693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
174382024-02-04T14:07:52.844Ze64868b6-a187-45e5-8f78-de858fbf62b3"}
174392024-02-04T14:07:52.844Z{"msg":"[1]R generation[0..12]: [160, 154, 161, 152, 161, 152, 160, 160, 156, 159, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.8662874Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174402024-02-04T14:07:52.844Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.8663403Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174412024-02-04T14:07:52.844Z{"msg":"[2]R flush_numbers[0..12]: [279, 267, 281, 264, 281, 262, 280, 280, 271, 277, 273, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866365574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174422024-02-04T14:07:52.844Z{"msg":"[2]R generation[0..12]: [160, 154, 161, 152, 161, 152, 160, 160, 156, 159, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866407552Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174432024-02-04T14:07:52.844Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866432776Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174442024-02-04T14:07:52.844Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866455266Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174452024-02-04T14:07:52.844Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866478478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174462024-02-04T14:07:52.844Z{"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866499964Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174472024-02-04T14:07:52.844Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866546651Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"":"downstairs","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174482024-02-04T14:07:52.844Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866570578Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174492024-02-04T14:07:52.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866591065Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"0","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174502024-02-04T14:07:52.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866645308Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"1","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174512024-02-04T14:07:52.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866668667Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"client":"2","":"downstairs","session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174522024-02-04T14:07:52.844Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866692394Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174532024-02-04T14:07:52.844Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866713506Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174542024-02-04T14:07:52.844Z{"msg":"96162f02-794c-44db-bcc7-953a139e4f58 is now active with session: e64868b6-a187-45e5-8f78-de858fbf62b3","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866755294Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174552024-02-04T14:07:52.844Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:51.866776437Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777,"session_id":"e64868b6-a187-45e5-8f78-de858fbf62b3"}
174562024-02-04T14:07:52.844Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174572024-02-04T14:07:52.845Z,"time":"2024-02-04T14:07:51.866819066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777}
174582024-02-04T14:07:52.845ZLoading write count information from file "/tmp/test_repair_verify.data"
174592024-02-04T14:07:52.845ZRun Repair workload
174602024-02-04T14:07:52.845Z01/30 Flush
174612024-02-04T14:07:52.845Z02/30 Flush
174622024-02-04T14:07:52.850Z03/30 Write block 458 len 36864 data: 17 17 16 15 12 13 13 16 18
174632024-02-04T14:07:52.850Z04/30 Read block 384 len 4096
174642024-02-04T14:07:52.851Z05/30 Read block 459 len 8192
174652024-02-04T14:07:52.851Z06/30 Write block 177 len 12288 data: 18 16 18
174662024-02-04T14:07:52.851Z07/30 Read block 222 len 40960
174672024-02-04T14:07:52.851Z08/30 Read block 216 len 8192
174682024-02-04T14:07:52.851Z09/30 Write block 34 len 16384 data: 12 15 16 14
174692024-02-04T14:07:52.851Z10/30 Read block 61 len 8192
174702024-02-04T14:07:52.851Z11/30 Read block 508 len 4096
174712024-02-04T14:07:52.854Z12/30 Write block 524 len 20480 data: 15 14 17 16 17
174722024-02-04T14:07:52.854Z13/30 Write block 451 len 32768 data: 12 14 13 15 16 15 17 18
174732024-02-04T14:07:52.854Z14/30 Read block 238 len 40960
174742024-02-04T14:07:52.859Z15/30 Read block 237 len 32768
174752024-02-04T14:07:52.859Z16/30 Write block 454 len 28672 data: 16 17 16 18 19 18 17
174762024-02-04T14:07:52.859Z17/30 Read block 286 len 36864
174772024-02-04T14:07:52.862Z18/30 Read block 394 len 4096
174782024-02-04T14:07:52.862Z19/30 Write block 8 len 4096 data: 11
174792024-02-04T14:07:52.867Z20/30 Write block 167 len 28672 data: 15 11 11 12 10 14 14
174802024-02-04T14:07:52.868Z21/30 Read block 31 len 36864
174812024-02-04T14:07:52.870Z22/30 Write block 280 len 8192 data: 9 8
174822024-02-04T14:07:52.870Z23/30 Write block 145 len 40960 data: 13 9 7 7 9 10 11 11 10 9
174832024-02-04T14:07:52.870Z24/30 Read block 243 len 4096
174842024-02-04T14:07:52.871Z25/30 Read block 3 len 12288
174852024-02-04T14:07:52.873Z26/30 Write block 230 len 32768 data: 11 11 11 12 15 12 15 18
174862024-02-04T14:07:52.873Z27/30 Write block 466 len 16384 data: 19 16 13 14
174872024-02-04T14:07:52.874Z28/30 Read block 365 len 40960
174882024-02-04T14:07:52.876Z29/30 Write block 239 len 36864 data: 14 17 15 12 11 12 12 14 15
174892024-02-04T14:07:52.880Z30/30 Read block 278 len 16384
174902024-02-04T14:07:52.880Z----------------------------------------------------------------
174912024-02-04T14:07:52.880Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:28
174922024-02-04T14:07:52.880ZStates: Active Active Active
174932024-02-04T14:07:52.880ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174942024-02-04T14:07:52.880Z 3 Acked 1002 Write 9 Done Done Done false
174952024-02-04T14:07:52.880Z 4 Acked 1003 Read 1 Done Done Done false
174962024-02-04T14:07:52.880Z 5 Acked 1004 Read 2 Sent Done Done false
174972024-02-04T14:07:52.880Z 6 Acked 1005 Write 3 Sent Done Done false
174982024-02-04T14:07:52.880Z 7 Acked 1006 Read 10 Sent Done Done false
174992024-02-04T14:07:52.880Z 8 Acked 1007 Read 2 Sent Done Done false
175002024-02-04T14:07:52.880Z 9 Acked 1008 Write 4 Sent Done Done false
175012024-02-04T14:07:52.880Z 10 Acked 1009 Read 2 Sent Done Done false
175022024-02-04T14:07:52.880Z 11 Acked 1010 Read 1 Sent Done Done false
175032024-02-04T14:07:52.880Z 12 Acked 1011 Write 5 Sent Done Done false
175042024-02-04T14:07:52.880Z 13 Acked 1012 Write 8 Sent Done Done false
175052024-02-04T14:07:52.880Z 14 Acked 1013 Read 10 Sent Done Done false
175062024-02-04T14:07:52.881Z 15 Acked 1014 Read 8 Sent Done Done false
175072024-02-04T14:07:52.881Z 16 Acked 1015 Write 7 Sent Done Done false
175082024-02-04T14:07:52.881Z 17 Acked 1016 Read 9 Sent Done Done false
175092024-02-04T14:07:52.881Z 18 Acked 1017 Read 1 Sent Done Done false
175102024-02-04T14:07:52.881Z 19 Acked 1018 Write 1 Sent Done Done false
175112024-02-04T14:07:52.881Z 20 Acked 1019 Write 7 Sent Done Done false
175122024-02-04T14:07:52.881Z 21 Acked 1020 Read 9 Sent Done Done false
175132024-02-04T14:07:52.881Z 22 Acked 1021 Write 2 Sent Done Done false
175142024-02-04T14:07:52.881Z 23 Acked 1022 Write 10 Sent Done Done false
175152024-02-04T14:07:52.881Z 24 Acked 1023 Read 1 Sent Done Done false
175162024-02-04T14:07:52.881Z 25 Acked 1024 Read 3 Sent Done Done false
175172024-02-04T14:07:52.881Z 26 Acked 1025 Write 8 Sent Done Done false
175182024-02-04T14:07:52.881Z 27 Acked 1026 Write 4 Sent Done Done false
175192024-02-04T14:07:52.881Z 28 Acked 1027 Read 10 Sent Done Done false
175202024-02-04T14:07:52.881Z 29 Acked 1028 Write 9 Sent Done Done false
175212024-02-04T14:07:52.881Z 30 Acked 1029 Read 4 Sent Done Done false
175222024-02-04T14:07:52.881Z STATES DS:0 DS:1 DS:2 TOTAL
175232024-02-04T14:07:52.881Z New 0 0 0 0
175242024-02-04T14:07:52.881Z Sent 26 0 0 26
175252024-02-04T14:07:52.881Z Done 2 28 28 58
175262024-02-04T14:07:52.881Z Skipped 0 0 0 0
175272024-02-04T14:07:52.881Z Error 0 0 0 0
175282024-02-04T14:07:52.881ZLast Flush: 1001 1001 1001
175292024-02-04T14:07:52.881ZDownstairs last five completed: 1001 1000
175302024-02-04T14:07:52.882ZUpstairs last five completed: 30 29 28 27 26
175312024-02-04T14:07:52.882Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:51.912114447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1777}
175322024-02-04T14:07:52.882ZWrote out file "/tmp/test_repair_verify.data"
175332024-02-04T14:07:52.886Z
175342024-02-04T14:07:52.893ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175352024-02-04T14:07:52.923ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175362024-02-04T14:07:52.923Z 0 000-019 160 160 160 279 279 279 F T T <---
175372024-02-04T14:07:52.923Z 1 020-039 154 154 154 267 267 267 F T T <---
175382024-02-04T14:07:52.923Z 2 040-059 161 161 161 281 281 281 F F F
175392024-02-04T14:07:52.923Z 3 060-079 152 152 152 264 264 264 F F F
175402024-02-04T14:07:52.923Z 4 080-099 161 161 161 281 281 281 F F F
175412024-02-04T14:07:52.923Z 5 100-119 152 152 152 262 262 262 F F F
175422024-02-04T14:07:52.923Z 6 120-139 160 160 160 280 280 280 F F F
175432024-02-04T14:07:52.923Z 7 140-159 160 160 160 280 280 280 F T T <---
175442024-02-04T14:07:52.923Z 8 160-179 156 156 156 271 271 271 F T T <---
175452024-02-04T14:07:52.923Z 9 180-199 159 159 159 277 277 277 F F F
175462024-02-04T14:07:52.923Z 10 200-219 158 158 158 273 273 273 F F F
175472024-02-04T14:07:52.924Z 11 220-239 158 158 158 275 275 275 F T T <---
175482024-02-04T14:07:52.924Z 12 240-259 157 157 157 272 272 272 F T T <---
175492024-02-04T14:07:52.924Z 13 260-279 160 160 160 280 280 280 F F F
175502024-02-04T14:07:52.924Z 14 280-299 152 152 152 263 263 263 F T T <---
175512024-02-04T14:07:52.924Z 15 300-319 160 160 160 280 280 280 F F F
175522024-02-04T14:07:52.924Z 16 320-339 161 161 161 281 281 281 F F F
175532024-02-04T14:07:52.924Z 17 340-359 156 156 156 269 269 269 F F F
175542024-02-04T14:07:52.927Z 18 360-379 160 160 160 280 280 280 F F F
175552024-02-04T14:07:52.927Z 19 380-399 154 154 154 267 267 267 F F F
175562024-02-04T14:07:52.927Z 20 400-419 159 159 159 277 277 277 F F F
175572024-02-04T14:07:52.927Z 21 420-439 161 161 161 281 281 281 F F F
175582024-02-04T14:07:52.927Z 22 440-459 161 161 161 281 281 281 T T T
175592024-02-04T14:07:52.927Z 23 460-479 158 158 158 275 275 275 T T T
175602024-02-04T14:07:52.927Z 24 480-499 160 160 160 280 280 280 F F F
175612024-02-04T14:07:52.927Z 25 500-519 160 160 160 279 279 279 F F F
175622024-02-04T14:07:52.927Z 26 520-539 159 159 159 277 277 277 F T T <---
175632024-02-04T14:07:52.927Z 27 540-559 159 159 159 277 277 277 F F F
175642024-02-04T14:07:52.927Z 28 560-579 159 159 159 277 277 277 F F F
175652024-02-04T14:07:52.927Z 29 580-599 124 124 124 205 205 205 F F F
175662024-02-04T14:07:52.927ZMax gen: 161, Max flush: 281
175672024-02-04T14:07:52.927ZError: Difference in extent metadata found!
175682024-02-04T14:07:52.931ZA Difference in extent metadata is expected here
175692024-02-04T14:07:52.931ZOn loop 81
175702024-02-04T14:07:52.931Z
175712024-02-04T14:07:52.933ZVerifying data now
175722024-02-04T14:07:59.018ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
175732024-02-04T14:07:59.022Z454,455d453
175742024-02-04T14:07:59.022Z< 11,
175752024-02-04T14:07:59.022Z< 13,
175762024-02-04T14:07:59.022Z458,459c456
175772024-02-04T14:07:59.022Z< 15,
175782024-02-04T14:07:59.022Z< 14,
175792024-02-04T14:07:59.022Z---
175802024-02-04T14:07:59.022Z> 13,
175812024-02-04T14:07:59.023Z462d458
175822024-02-04T14:07:59.023Z< 17,
175832024-02-04T14:07:59.023Z463a460,463
175842024-02-04T14:07:59.023Z> 18,
175852024-02-04T14:07:59.023Z> 19,
175862024-02-04T14:07:59.023Z> 18,
175872024-02-04T14:07:59.023Z> 17,
175882024-02-04T14:07:59.023Z469,471c469,470
175892024-02-04T14:07:59.023Z< 18,
175902024-02-04T14:07:59.023Z< 15,
175912024-02-04T14:07:59.023Z< 12,
175922024-02-04T14:07:59.024Z---
175932024-02-04T14:07:59.024Z> 19,
175942024-02-04T14:07:59.024Z> 16,
175952024-02-04T14:07:59.024Z472a472
175962024-02-04T14:07:59.024Z> 14,
175972024-02-04T14:07:59.024Zdiff found after verify
175982024-02-04T14:07:59.030ZLoop: 81 Downstairs dump after verify (and repair):
175992024-02-04T14:07:59.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176002024-02-04T14:07:59.057Z 0 000-019 162 162 162 282 282 282 F F F
176012024-02-04T14:07:59.057Z 1 020-039 162 162 162 282 282 282 F F F
176022024-02-04T14:07:59.057Z 2 040-059 161 161 161 281 281 281 F F F
176032024-02-04T14:07:59.057Z 3 060-079 152 152 152 264 264 264 F F F
176042024-02-04T14:07:59.057Z 4 080-099 161 161 161 281 281 281 F F F
176052024-02-04T14:07:59.057Z 5 100-119 152 152 152 262 262 262 F F F
176062024-02-04T14:07:59.057Z 6 120-139 160 160 160 280 280 280 F F F
176072024-02-04T14:07:59.057Z 7 140-159 162 162 162 282 282 282 F F F
176082024-02-04T14:07:59.057Z 8 160-179 162 162 162 282 282 282 F F F
176092024-02-04T14:07:59.057Z 9 180-199 159 159 159 277 277 277 F F F
176102024-02-04T14:07:59.057Z 10 200-219 158 158 158 273 273 273 F F F
176112024-02-04T14:07:59.064Z 11 220-239 162 162 162 282 282 282 F F F
176122024-02-04T14:07:59.064Z 12 240-259 162 162 162 282 282 282 F F F
176132024-02-04T14:07:59.064Z 13 260-279 160 160 160 280 280 280 F F F
176142024-02-04T14:07:59.064Z 14 280-299 162 162 162 282 282 282 F F F
176152024-02-04T14:07:59.064Z 15 300-319 160 160 160 280 280 280 F F F
176162024-02-04T14:07:59.064Z 16 320-339 161 161 161 281 281 281 F F F
176172024-02-04T14:07:59.065Z 17 340-359 156 156 156 269 269 269 F F F
176182024-02-04T14:07:59.065Z 18 360-379 160 160 160 280 280 280 F F F
176192024-02-04T14:07:59.065Z 19 380-399 154 154 154 267 267 267 F F F
176202024-02-04T14:07:59.065Z 20 400-419 159 159 159 277 277 277 F F F
176212024-02-04T14:07:59.065Z 21 420-439 161 161 161 281 281 281 F F F
176222024-02-04T14:07:59.065Z 22 440-459 162 162 162 282 282 282 F F F
176232024-02-04T14:07:59.065Z 23 460-479 162 162 162 282 282 282 F F F
176242024-02-04T14:07:59.065Z 24 480-499 160 160 160 280 280 280 F F F
176252024-02-04T14:07:59.065Z 25 500-519 160 160 160 279 279 279 F F F
176262024-02-04T14:07:59.065Z 26 520-539 162 162 162 282 282 282 F F F
176272024-02-04T14:07:59.065Z 27 540-559 159 159 159 277 277 277 F F F
176282024-02-04T14:07:59.065Z 28 560-579 159 159 159 277 277 277 F F F
176292024-02-04T14:07:59.065Z 29 580-599 124 124 124 205 205 205 F F F
176302024-02-04T14:07:59.065ZMax gen: 162, Max flush: 282
176312024-02-04T14:07:59.065Z
176322024-02-04T14:07:59.065ZBegin loop 82
176332024-02-04T14:07:59.065ZDownstairs to restart: 1
176342024-02-04T14:07:59.088Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.117611981Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788}
176352024-02-04T14:07:59.088Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118147861Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788}
176362024-02-04T14:07:59.088Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
176372024-02-04T14:07:59.088Z,"time":"2024-02-04T14:07:58.118224669Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788}
176382024-02-04T14:07:59.088Z{"msg":"Crucible 432bf28f-f210-43cf-ae5f-c3dc4f44a897 has session id: 499f699b-396c-45eb-bac1-9c97802ebbff","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118277045Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176392024-02-04T14:07:59.088Z{"msg":"Upstairs opts: Upstairs UUID: 432bf28f-f210-43cf-ae5f-c3dc4f44a897, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118343395Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176402024-02-04T14:07:59.088Z{"msg":"Crucible stats registered with UUID: 432bf28f-f210-43cf-ae5f-c3dc4f44a897","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118371827Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176412024-02-04T14:07:59.089Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118413559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788}
176422024-02-04T14:07:59.089Z{"msg":"432bf28f-f210-43cf-ae5f-c3dc4f44a897 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118542873Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176432024-02-04T14:07:59.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118654567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176442024-02-04T14:07:59.089Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118727775Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176452024-02-04T14:07:59.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118775396Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176462024-02-04T14:07:59.089Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118850997Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176472024-02-04T14:07:59.089Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.118895414Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176482024-02-04T14:07:59.089Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.11893736Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176492024-02-04T14:07:59.089Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.119041312Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176502024-02-04T14:07:59.089Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.119118466Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176512024-02-04T14:07:59.089Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.119168043Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176522024-02-04T14:07:59.089Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.119240254Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176532024-02-04T14:07:59.089Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.119289136Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176542024-02-04T14:07:59.090Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.119329746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176552024-02-04T14:07:59.094Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.120437274Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176562024-02-04T14:07:59.095Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.120550791Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176572024-02-04T14:07:59.095Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.120596513Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176582024-02-04T14:07:59.095Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.120642942Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176592024-02-04T14:07:59.095Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.121502318Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176602024-02-04T14:07:59.095Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.121743416Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176612024-02-04T14:07:59.095Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.121802706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176622024-02-04T14:07:59.095Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.121829863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176632024-02-04T14:07:59.095Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.122267211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176642024-02-04T14:07:59.095Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.122336572Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176652024-02-04T14:07:59.095Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.122363371Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176662024-02-04T14:07:59.095Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.12271083Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176672024-02-04T14:07:59.095Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.12278319Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176682024-02-04T14:07:59.096Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.122809043Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176692024-02-04T14:07:59.096Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.12293579Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176702024-02-04T14:07:59.096Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.123817148Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176712024-02-04T14:07:59.096Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.123875319Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176722024-02-04T14:07:59.096Z{"msg":"[0]R flush_numbers[0..12]: [282, 282, 281, 264, 281, 262, 280, 282, 282, 277, 273, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.123901963Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176732024-02-04T14:07:59.096Z{"msg":"[0]R generation[0..12]: [162, 162, 161, 152, 161, 152, 160, 162, 162, 159, 158, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.123945235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176742024-02-04T14:07:59.096Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
176752024-02-04T14:07:59.096Z30,"time":"2024-02-04T14:07:58.123975721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176762024-02-04T14:07:59.096Z{"msg":"[1]R flush_numbers[0..12]: [282, 282, 281, 264, 281, 262, 280, 282, 282, 277, 273, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124000973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176772024-02-04T14:07:59.096Z{"msg":"[1]R generation[0..12]: [162, 162, 161, 152, 161, 152, 160, 162, 162, 159, 158, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124025004Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176782024-02-04T14:07:59.096Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124049225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176792024-02-04T14:07:59.096Z{"msg":"[2]R flush_numbers[0..12]: [282, 282, 281, 264, 281, 262, 280, 282, 282, 277, 273, 282]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124073152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176802024-02-04T14:07:59.097Z{"msg":"[2]R generation[0..12]: [162, 162, 161, 152, 161, 152, 160, 162, 162, 159, 158, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124096559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176812024-02-04T14:07:59.097Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124120573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176822024-02-04T14:07:59.097Z{"msg":"Max found gen is 163","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124142995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176832024-02-04T14:07:59.097Z{"msg":"Generation requested: 164 >= found:163","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124186992Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176842024-02-04T14:07:59.097Z{"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124210165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176852024-02-04T14:07:59.097Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124232236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176862024-02-04T14:07:59.097Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124254845Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176872024-02-04T14:07:59.097Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124275076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176882024-02-04T14:07:59.097Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124297488Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176892024-02-04T14:07:59.097Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124344187Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176902024-02-04T14:07:59.097Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124367928Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176912024-02-04T14:07:59.097Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.12441397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176922024-02-04T14:07:59.098Z{"msg":"432bf28f-f210-43cf-ae5f-c3dc4f44a897 is now active with session: 499f699b-396c-45eb-bac1-9c97802ebbff","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124437279Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176932024-02-04T14:07:59.098Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124457409Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788,"session_id":"499f699b-396c-45eb-bac1-9c97802ebbff"}
176942024-02-04T14:07:59.098Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:07:58.124499769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788}
176952024-02-04T14:07:59.098ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176962024-02-04T14:07:59.098ZLoading write count information from file "/tmp/test_repair_verify.data"
176972024-02-04T14:07:59.098ZRun Repair workload
176982024-02-04T14:07:59.102Z01/30 Write block 586 len 12288 data: 14 12 13
176992024-02-04T14:07:59.102Z02/30 Write block 541 len 24576 data: 13 12 15 13 12 9
177002024-02-04T14:07:59.102Z03/30 Read block 121 len 16384
177012024-02-04T14:07:59.102Z04/30 Read block 146 len 16384
177022024-02-04T14:07:59.102Z05/30 Read block 0 len 12288
177032024-02-04T14:07:59.102Z06/30 Read block 237 len 36864
177042024-02-04T14:07:59.105Z07/30 Read block 336 len 32768
177052024-02-04T14:07:59.105Z08/30 Read block 542 len 20480
177062024-02-04T14:07:59.109Z09/30 Write block 44 len 40960 data: 11 11 11 10 12 9 8 11 12 12
177072024-02-04T14:07:59.109Z10/30 Read block 37 len 28672
177082024-02-04T14:07:59.112Z11/30 Read block 97 len 32768
177092024-02-04T14:07:59.112Z12/30 Write block 225 len 36864 data: 10 11 11 14 12 12 12 12 13
177102024-02-04T14:07:59.116Z13/30 Read block 473 len 24576
177112024-02-04T14:07:59.116Z14/30 Flush
177122024-02-04T14:07:59.121Z15/30 Write block 467 len 40960 data: 16 13 14 12 10 11 11 10 11 10
177132024-02-04T14:07:59.121Z16/30 Flush
177142024-02-04T14:07:59.124Z17/30 Write block 403 len 16384 data: 13 14 16 14
177152024-02-04T14:07:59.127Z18/30 Write block 519 len 40960 data: 11 7 9 12 13 16 15 18 17 18
177162024-02-04T14:07:59.127Z19/30 Write block 67 len 32768 data: 13 13 11 9 10 10 10 10
177172024-02-04T14:07:59.130Z20/30 Read block 226 len 20480
177182024-02-04T14:07:59.130Z21/30 Read block 139 len 4096
177192024-02-04T14:07:59.133Z22/30 Write block 264 len 40960 data: 13 14 15 12 15 14 14 12 8 9
177202024-02-04T14:07:59.133Z23/30 Write block 248 len 20480 data: 19 18 18 17 15
177212024-02-04T14:07:59.136Z24/30 Write block 45 len 32768 data: 12 12 11 13 10 9 12 13
177222024-02-04T14:07:59.136Z25/30 Write block 63 len 4096 data: 13
177232024-02-04T14:07:59.136Z26/30 Write block 535 len 32768 data: 7 8 6 9 11 13 14 13
177242024-02-04T14:07:59.151Z27/30 Write block 305 len 16384 data: 14 13 13 12
177252024-02-04T14:07:59.151Z28/30 Read block 274 len 40960
177262024-02-04T14:07:59.155Z29/30 Read block 185 len 40960
177272024-02-04T14:07:59.155Z30/30 Write block 323 len 4096 data: 9
177282024-02-04T14:07:59.155Z----------------------------------------------------------------
177292024-02-04T14:07:59.156Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
177302024-02-04T14:07:59.156ZStates: Active Active Active
177312024-02-04T14:07:59.156ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177322024-02-04T14:07:59.156Z 1 Acked 1000 Write 3 Done Sent Done false
177332024-02-04T14:07:59.156Z 2 Acked 1001 Write 6 Done Sent Done false
177342024-02-04T14:07:59.156Z 3 Acked 1002 Read 4 Done Sent Done false
177352024-02-04T14:07:59.156Z 4 Acked 1003 Read 4 Done Sent Done false
177362024-02-04T14:07:59.156Z 5 Acked 1004 Read 3 Done Sent Done false
177372024-02-04T14:07:59.156Z 6 Acked 1005 Read 9 Done Sent Done false
177382024-02-04T14:07:59.156Z 7 Acked 1006 Read 8 Done Sent Done false
177392024-02-04T14:07:59.156Z 8 Acked 1007 Read 5 Done Sent Done false
177402024-02-04T14:07:59.156Z 9 Acked 1008 Write 10 Done Sent Done false
177412024-02-04T14:07:59.156Z 10 Acked 1009 Read 7 Done Sent Done false
177422024-02-04T14:07:59.157Z 11 Acked 1010 Read 8 Done Sent Done false
177432024-02-04T14:07:59.157Z 12 Acked 1011 Write 9 Done Sent Done false
177442024-02-04T14:07:59.157Z 13 Acked 1012 Read 6 Done Sent Done false
177452024-02-04T14:07:59.157Z 14 Acked 1013 Flush 0 Done Sent Done false
177462024-02-04T14:07:59.157Z 15 Acked 1014 Write 10 Done Sent Done false
177472024-02-04T14:07:59.157Z 16 Acked 1015 Flush 0 Done Sent Done false
177482024-02-04T14:07:59.157Z 17 Acked 1016 Write 4 Done Sent Done false
177492024-02-04T14:07:59.157Z 18 Acked 1017 Write 10 Done Sent Done false
177502024-02-04T14:07:59.157Z 19 Acked 1018 Write 8 Done Sent Done false
177512024-02-04T14:07:59.157Z 20 Acked 1019 Read 5 Done Sent Done false
177522024-02-04T14:07:59.157Z 21 Acked 1020 Read 1 Done Sent Done false
177532024-02-04T14:07:59.157Z 22 Acked 1021 Write 10 Done Sent Done false
177542024-02-04T14:07:59.157Z 23 Acked 1022 Write 5 Done Sent Done false
177552024-02-04T14:07:59.158Z 24 Acked 1023 Write 8 Done Sent Done false
177562024-02-04T14:07:59.158Z 25 Acked 1024 Write 1 Done Sent Done false
177572024-02-04T14:07:59.158Z 26 Acked 1025 Write 8 Done Sent Done false
177582024-02-04T14:07:59.158Z 27 Acked 1026 Write 4 Done Sent Done false
177592024-02-04T14:07:59.158Z 28 Acked 1027 Read 10 Done Sent Done false
177602024-02-04T14:07:59.158Z 29 Acked 1028 Read 10 Done Sent Done false
177612024-02-04T14:07:59.158Z 30 Acked 1029 Write 1 Sent Sent Sent false
177622024-02-04T14:07:59.158Z STATES DS:0 DS:1 DS:2 TOTAL
177632024-02-04T14:07:59.158Z New 0 0 0 0
177642024-02-04T14:07:59.158Z Sent 1 30 1 32
177652024-02-04T14:07:59.158Z Done 29 0 29 58
177662024-02-04T14:07:59.158Z Skipped 0 0 0 0
177672024-02-04T14:07:59.158Z Error 0 0 0 0
177682024-02-04T14:07:59.159ZLast Flush: 1015 0 1015
177692024-02-04T14:07:59.159ZDownstairs last five completed:
177702024-02-04T14:07:59.159ZUpstairs last five completed: 30 29 28 27 26
177712024-02-04T14:07:59.159Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:07:58.176115493Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1788}
177722024-02-04T14:07:59.159ZWrote out file "/tmp/test_repair_verify.data"
177732024-02-04T14:07:59.159Z
177742024-02-04T14:07:59.162ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177752024-02-04T14:07:59.184ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177762024-02-04T14:07:59.184Z 0 000-019 162 162 162 282 282 282 F F F
177772024-02-04T14:07:59.184Z 1 020-039 162 162 162 282 282 282 F F F
177782024-02-04T14:07:59.184Z 2 040-059 164 161 164 283 281 283 T F T <---
177792024-02-04T14:07:59.184Z 3 060-079 152 152 152 264 264 264 T F T <---
177802024-02-04T14:07:59.184Z 4 080-099 161 161 161 281 281 281 F F F
177812024-02-04T14:07:59.184Z 5 100-119 152 152 152 262 262 262 F F F
177822024-02-04T14:07:59.184Z 6 120-139 160 160 160 280 280 280 F F F
177832024-02-04T14:07:59.188Z 7 140-159 162 162 162 282 282 282 F F F
177842024-02-04T14:07:59.188Z 8 160-179 162 162 162 282 282 282 F F F
177852024-02-04T14:07:59.188Z 9 180-199 159 159 159 277 277 277 F F F
177862024-02-04T14:07:59.188Z 10 200-219 158 158 158 273 273 273 F F F
177872024-02-04T14:07:59.188Z 11 220-239 164 162 164 283 282 283 F F F <---
177882024-02-04T14:07:59.188Z 12 240-259 162 162 162 282 282 282 T F T <---
177892024-02-04T14:07:59.188Z 13 260-279 160 160 160 280 280 280 T F T <---
177902024-02-04T14:07:59.188Z 14 280-299 162 162 162 282 282 282 F F F
177912024-02-04T14:07:59.188Z 15 300-319 160 160 160 280 280 280 T F T <---
177922024-02-04T14:07:59.188Z 16 320-339 161 161 161 281 281 281 T F T <---
177932024-02-04T14:07:59.188Z 17 340-359 156 156 156 269 269 269 F F F
177942024-02-04T14:07:59.188Z 18 360-379 160 160 160 280 280 280 F F F
177952024-02-04T14:07:59.189Z 19 380-399 154 154 154 267 267 267 F F F
177962024-02-04T14:07:59.189Z 20 400-419 159 159 159 277 277 277 T F T <---
177972024-02-04T14:07:59.189Z 21 420-439 161 161 161 281 281 281 F F F
177982024-02-04T14:07:59.189Z 22 440-459 162 162 162 282 282 282 F F F
177992024-02-04T14:07:59.189Z 23 460-479 164 162 164 284 282 284 F F F <---
178002024-02-04T14:07:59.189Z 24 480-499 160 160 160 280 280 280 F F F
178012024-02-04T14:07:59.189Z 25 500-519 160 160 160 279 279 279 T F T <---
178022024-02-04T14:07:59.189Z 26 520-539 162 162 162 282 282 282 T F T <---
178032024-02-04T14:07:59.189Z 27 540-559 164 159 164 283 277 283 T F T <---
178042024-02-04T14:07:59.189Z 28 560-579 159 159 159 277 277 277 F F F
178052024-02-04T14:07:59.189Z 29 580-599 164 124 164 283 205 283 F F F <---
178062024-02-04T14:07:59.189ZMax gen: 164, Max flush: 284
178072024-02-04T14:07:59.189ZError: Difference in extent metadata found!
178082024-02-04T14:07:59.193ZA Difference in extent metadata is expected here
178092024-02-04T14:07:59.193ZOn loop 82
178102024-02-04T14:07:59.193Z
178112024-02-04T14:07:59.193ZVerifying data now
178122024-02-04T14:08:04.338ZNo change after verify
178132024-02-04T14:08:04.338ZLoop: 82 Downstairs dump after verify (and repair):
178142024-02-04T14:08:04.363ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178152024-02-04T14:08:04.364Z 0 000-019 162 162 162 282 282 282 F F F
178162024-02-04T14:08:04.364Z 1 020-039 162 162 162 282 282 282 F F F
178172024-02-04T14:08:04.364Z 2 040-059 165 165 165 285 285 285 F F F
178182024-02-04T14:08:04.364Z 3 060-079 165 165 165 285 285 285 F F F
178192024-02-04T14:08:04.364Z 4 080-099 161 161 161 281 281 281 F F F
178202024-02-04T14:08:04.367Z 5 100-119 152 152 152 262 262 262 F F F
178212024-02-04T14:08:04.367Z 6 120-139 160 160 160 280 280 280 F F F
178222024-02-04T14:08:04.367Z 7 140-159 162 162 162 282 282 282 F F F
178232024-02-04T14:08:04.367Z 8 160-179 162 162 162 282 282 282 F F F
178242024-02-04T14:08:04.367Z 9 180-199 159 159 159 277 277 277 F F F
178252024-02-04T14:08:04.367Z 10 200-219 158 158 158 273 273 273 F F F
178262024-02-04T14:08:04.368Z 11 220-239 164 164 164 283 283 283 F F F
178272024-02-04T14:08:04.368Z 12 240-259 165 165 165 285 285 285 F F F
178282024-02-04T14:08:04.368Z 13 260-279 165 165 165 285 285 285 F F F
178292024-02-04T14:08:04.368Z 14 280-299 162 162 162 282 282 282 F F F
178302024-02-04T14:08:04.368Z 15 300-319 165 165 165 285 285 285 F F F
178312024-02-04T14:08:04.368Z 16 320-339 165 165 165 285 285 285 F F F
178322024-02-04T14:08:04.368Z 17 340-359 156 156 156 269 269 269 F F F
178332024-02-04T14:08:04.368Z 18 360-379 160 160 160 280 280 280 F F F
178342024-02-04T14:08:04.368Z 19 380-399 154 154 154 267 267 267 F F F
178352024-02-04T14:08:04.368Z 20 400-419 165 165 165 285 285 285 F F F
178362024-02-04T14:08:04.368Z 21 420-439 161 161 161 281 281 281 F F F
178372024-02-04T14:08:04.368Z 22 440-459 162 162 162 282 282 282 F F F
178382024-02-04T14:08:04.368Z 23 460-479 164 164 164 284 284 284 F F F
178392024-02-04T14:08:04.368Z 24 480-499 160 160 160 280 280 280 F F F
178402024-02-04T14:08:04.368Z 25 500-519 165 165 165 285 285 285 F F F
178412024-02-04T14:08:04.368Z 26 520-539 165 165 165 285 285 285 F F F
178422024-02-04T14:08:04.368Z 27 540-559 165 165 165 285 285 285 F F F
178432024-02-04T14:08:04.368Z 28 560-579 159 159 159 277 277 277 F F F
178442024-02-04T14:08:04.368Z 29 580-599 164 164 164 283 283 283 F F F
178452024-02-04T14:08:04.368ZMax gen: 165, Max flush: 285
178462024-02-04T14:08:04.368Z
178472024-02-04T14:08:04.368ZBegin loop 83
178482024-02-04T14:08:04.368ZDownstairs to restart: 1
178492024-02-04T14:08:04.396Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.427846925Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796}
178502024-02-04T14:08:04.396Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.42837305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796}
178512024-02-04T14:08:04.400Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.428425219Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796}
178522024-02-04T14:08:04.400Z{"msg":"Crucible d8f715ef-8d13-4c0f-801c-c44a18dbfc26 has session id: ee07f6d6-8428-4b67-a116-7e68182ce920","vCrucible runtime is spawned
178532024-02-04T14:08:04.400Z":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.428579429Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178542024-02-04T14:08:04.401Z{"msg":"Upstairs opts: Upstairs UUID: d8f715ef-8d13-4c0f-801c-c44a18dbfc26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.428653092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178552024-02-04T14:08:04.401Z{"msg":"Crucible stats registered with UUID: d8f715ef-8d13-4c0f-801c-c44a18dbfc26","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.428680815Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178562024-02-04T14:08:04.401Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.428724568Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796}
178572024-02-04T14:08:04.401Z{"msg":"d8f715ef-8d13-4c0f-801c-c44a18dbfc26 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.428752637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178582024-02-04T14:08:04.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.428775475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178592024-02-04T14:08:04.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.42896865Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178602024-02-04T14:08:04.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429071774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178612024-02-04T14:08:04.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429121509Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178622024-02-04T14:08:04.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429193231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178632024-02-04T14:08:04.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429301014Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178642024-02-04T14:08:04.401Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429356386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178652024-02-04T14:08:04.401Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429401963Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178662024-02-04T14:08:04.401Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429472552Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178672024-02-04T14:08:04.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429516894Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178682024-02-04T14:08:04.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429744914Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178692024-02-04T14:08:04.401Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.429835881Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178702024-02-04T14:08:04.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.430467305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178712024-02-04T14:08:04.401Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.430526656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178722024-02-04T14:08:04.401Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.430790747Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178732024-02-04T14:08:04.401Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.430929359Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178742024-02-04T14:08:04.401Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.431497353Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178752024-02-04T14:08:04.401Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.431766172Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178762024-02-04T14:08:04.408Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.432378739Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178772024-02-04T14:08:04.408Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.432460059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178782024-02-04T14:08:04.408Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.432490724Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178792024-02-04T14:08:04.408Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.432563833Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178802024-02-04T14:08:04.409Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.432590489Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178812024-02-04T14:08:04.409Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.432810313Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178822024-02-04T14:08:04.409Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.432912695Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178832024-02-04T14:08:04.409Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.432977178Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178842024-02-04T14:08:04.409Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.433453377Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178852024-02-04T14:08:04.409Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434161799Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178862024-02-04T14:08:04.409Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434294055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178872024-02-04T14:08:04.409Z{"msg":"[0]R flush_numbers[0..12]: [282, 282, 285, 285, 281, 262, 280, 282, 282, 277, 273, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434347575Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178882024-02-04T14:08:04.409Z{"msg":"[0]R generation[0..12]: [162, 162, 165, 165, 161, 152, 160, 162, 162, 159, 158, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434373264Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178892024-02-04T14:08:04.409Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434416738Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178902024-02-04T14:08:04.409Z{"msg":"[1]R flush_numbers[0..12]: [282, 282, 285, 285, 281, 262, 280, 282, 282, 277, 273, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434442081Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178912024-02-04T14:08:04.409Z{"msg":"[1]R generation[0..12]: [162, 162, 165, 165, 161, 152, 160, 162, 162, 159, 158, 164]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
178922024-02-04T14:08:04.409Z,"time":"2024-02-04T14:08:03.434465419Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178932024-02-04T14:08:04.409Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434508012Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178942024-02-04T14:08:04.409Z{"msg":"[2]R flush_numbers[0..12]: [282, 282, 285, 285, 281, 262, 280, 282, 282, 277, 273, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434566523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178952024-02-04T14:08:04.409Z{"msg":"[2]R generation[0..12]: [162, 162, 165, 165, 161, 152, 160, 162, 162, 159, 158, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.43459336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178962024-02-04T14:08:04.409Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434618279Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178972024-02-04T14:08:04.409Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.43464188Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178982024-02-04T14:08:04.409Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434705487Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
178992024-02-04T14:08:04.409Z{"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434728997Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179002024-02-04T14:08:04.409Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434751357Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179012024-02-04T14:08:04.409Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.43479231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179022024-02-04T14:08:04.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434815885Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179032024-02-04T14:08:04.409Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434838782Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179042024-02-04T14:08:04.410Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434861277Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179052024-02-04T14:08:04.410Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434883702Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179062024-02-04T14:08:04.410Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434931034Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179072024-02-04T14:08:04.410Z{"msg":"d8f715ef-8d13-4c0f-801c-c44a18dbfc26 is now active with session: ee07f6d6-8428-4b67-a116-7e68182ce920","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.434961512Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179082024-02-04T14:08:04.410Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.435027141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796,"session_id":"ee07f6d6-8428-4b67-a116-7e68182ce920"}
179092024-02-04T14:08:04.410Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:03.435055995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796}
179102024-02-04T14:08:04.410ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
179112024-02-04T14:08:04.410ZLoading write count information from file "/tmp/test_repair_verify.data"
179122024-02-04T14:08:04.410ZRun Repair workload
179132024-02-04T14:08:04.410Z01/30 Write block 276 len 16384 data: 8 8 10 9
179142024-02-04T14:08:04.410Z02/30 Read block 93 len 8192
179152024-02-04T14:08:04.413Z03/30 Write block 508 len 20480 data: 16 16 14 12 7
179162024-02-04T14:08:04.413Z04/30 Write block 136 len 4096 data: 8
179172024-02-04T14:08:04.413Z05/30 Write block 90 len 32768 data: 5 7 8 7 8 10 9 9
179182024-02-04T14:08:04.413Z06/30 Write block 220 len 28672 data: 11 12 9 9 8 11 12
179192024-02-04T14:08:04.414Z07/30 Write block 136 len 36864 data: 9 8 9 11 16 12 8 9 12
179202024-02-04T14:08:04.414Z08/30 Read block 524 len 16384
179212024-02-04T14:08:04.417Z09/30 Read block 109 len 4096
179222024-02-04T14:08:04.417Z10/30 Write block 371 len 8192 data: 12 8
179232024-02-04T14:08:04.417Z11/30 Read block 92 len 8192
179242024-02-04T14:08:04.420Z12/30 Read block 2 len 16384
179252024-02-04T14:08:04.420Z13/30 Read block 226 len 4096
179262024-02-04T14:08:04.424Z14/30 Write block 22 len 8192 data: 14 15
179272024-02-04T14:08:04.424Z15/30 Read block 590 len 32768
179282024-02-04T14:08:04.424Z16/30 Write block 288 len 4096 data: 16
179292024-02-04T14:08:04.424Z17/30 Read block 473 len 24576
179302024-02-04T14:08:04.428Z18/30 Write block 205 len 4096 data: 15
179312024-02-04T14:08:04.428Z19/30 Read block 351 len 8192
179322024-02-04T14:08:04.428Z20/30 Write block 119 len 36864 data: 12 12 14 13 11 12 14 14 12
179332024-02-04T14:08:04.431Z21/30 Write block 411 len 12288 data: 14 15 15
179342024-02-04T14:08:04.431Z22/30 Read block 516 len 24576
179352024-02-04T14:08:04.431Z23/30 Write block 37 len 28672 data: 15 12 11 10 10 13 11
179362024-02-04T14:08:04.435Z24/30 Write block 259 len 24576 data: 12 11 13 15 15 14
179372024-02-04T14:08:04.435Z25/30 Flush
179382024-02-04T14:08:04.440Z26/30 Write block 123 len 40960 data: 12 13 15 15 13 10 8 8 7 10
179392024-02-04T14:08:04.445Z27/30 Read block 23 len 36864
179402024-02-04T14:08:04.445Z28/30 Write block 64 len 4096 data: 10
179412024-02-04T14:08:04.445Z29/30 Read block 546 len 24576
179422024-02-04T14:08:04.451Z30/30 Read block 234 len 36864
179432024-02-04T14:08:04.451Z----------------------------------------------------------------
179442024-02-04T14:08:04.451Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
179452024-02-04T14:08:04.451ZStates: Active Active Active
179462024-02-04T14:08:04.451ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179472024-02-04T14:08:04.451Z 1 Acked 1000 Write 4 Done Done Done false
179482024-02-04T14:08:04.451Z 2 Acked 1001 Read 2 Done Done Done false
179492024-02-04T14:08:04.451Z 3 Acked 1002 Write 5 Done Sent Done false
179502024-02-04T14:08:04.451Z 4 Acked 1003 Write 1 Done Sent Done false
179512024-02-04T14:08:04.451Z 5 Acked 1004 Write 8 Done Sent Done false
179522024-02-04T14:08:04.451Z 6 Acked 1005 Write 7 Done Sent Done false
179532024-02-04T14:08:04.452Z 7 Acked 1006 Write 9 Done Sent Done false
179542024-02-04T14:08:04.452Z 8 Acked 1007 Read 4 Done Sent Done false
179552024-02-04T14:08:04.452Z 9 Acked 1008 Read 1 Done Sent Done false
179562024-02-04T14:08:04.452Z 10 Acked 1009 Write 2 Done Sent Done false
179572024-02-04T14:08:04.452Z 11 Acked 1010 Read 2 Done Sent Done false
179582024-02-04T14:08:04.452Z 12 Acked 1011 Read 4 Done Sent Done false
179592024-02-04T14:08:04.452Z 13 Acked 1012 Read 1 Done Sent Done false
179602024-02-04T14:08:04.452Z 14 Acked 1013 Write 2 Done Sent Done false
179612024-02-04T14:08:04.452Z 15 Acked 1014 Read 8 Done Sent Done false
179622024-02-04T14:08:04.452Z 16 Acked 1015 Write 1 Done Sent Done false
179632024-02-04T14:08:04.452Z 17 Acked 1016 Read 6 Done Sent Done false
179642024-02-04T14:08:04.452Z 18 Acked 1017 Write 1 Done Sent Done false
179652024-02-04T14:08:04.452Z 19 Acked 1018 Read 2 Done Sent Done false
179662024-02-04T14:08:04.452Z 20 Acked 1019 Write 9 Done Sent Done false
179672024-02-04T14:08:04.453Z 21 Acked 1020 Write 3 Done Sent Done false
179682024-02-04T14:08:04.453Z 22 Acked 1021 Read 6 Done Sent Done false
179692024-02-04T14:08:04.453Z 23 Acked 1022 Write 7 Done Sent Done false
179702024-02-04T14:08:04.453Z 24 Acked 1023 Write 6 Done Sent Done false
179712024-02-04T14:08:04.453Z 25 Acked 1024 Flush 0 Done Sent Done false
179722024-02-04T14:08:04.453Z 26 Acked 1025 Write 10 Done Sent Done false
179732024-02-04T14:08:04.453Z 27 Acked 1026 Read 9 Done Sent Done false
179742024-02-04T14:08:04.453Z 28 Acked 1027 Write 1 Done Sent Done false
179752024-02-04T14:08:04.453Z 29 Acked 1028 Read 6 Done Sent Done false
179762024-02-04T14:08:04.453Z 30 Acked 1029 Read 9 Done Sent Sent false
179772024-02-04T14:08:04.453Z STATES DS:0 DS:1 DS:2 TOTAL
179782024-02-04T14:08:04.453Z New 0 0 0 0
179792024-02-04T14:08:04.453Z Sent 0 28 1 29
179802024-02-04T14:08:04.453Z Done 30 2 29 61
179812024-02-04T14:08:04.454Z Skipped 0 0 0 0
179822024-02-04T14:08:04.454Z Error 0 0 0 0
179832024-02-04T14:08:04.454ZLast Flush: 1024 0 1024
179842024-02-04T14:08:04.454ZDownstairs last five completed:
179852024-02-04T14:08:04.454ZUpstairs last five completed: 30 29 28 27 26
179862024-02-04T14:08:04.454Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:03.480006277Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1796}
179872024-02-04T14:08:04.454ZWrote out file "/tmp/test_repair_verify.data"
179882024-02-04T14:08:04.458Z
179892024-02-04T14:08:04.474ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179902024-02-04T14:08:04.484ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179912024-02-04T14:08:04.484Z 0 000-019 162 162 162 282 282 282 F F F
179922024-02-04T14:08:04.484Z 1 020-039 166 162 166 286 282 286 F F F <---
179932024-02-04T14:08:04.488Z 2 040-059 166 165 166 286 285 286 F F F <---
179942024-02-04T14:08:04.488Z 3 060-079 165 165 165 285 285 285 T F T <---
179952024-02-04T14:08:04.488Z 4 080-099 166 161 166 286 281 286 F F F <---
179962024-02-04T14:08:04.488Z 5 100-119 166 152 166 286 262 286 F F F <---
179972024-02-04T14:08:04.488Z 6 120-139 166 160 166 286 280 286 T F T <---
179982024-02-04T14:08:04.488Z 7 140-159 166 162 166 286 282 286 F F F <---
179992024-02-04T14:08:04.488Z 8 160-179 162 162 162 282 282 282 F F F
180002024-02-04T14:08:04.488Z 9 180-199 159 159 159 277 277 277 F F F
180012024-02-04T14:08:04.488Z 10 200-219 166 158 166 286 273 286 F F F <---
180022024-02-04T14:08:04.488Z 11 220-239 166 164 166 286 283 286 F F F <---
180032024-02-04T14:08:04.488Z 12 240-259 166 165 166 286 285 286 F F F <---
180042024-02-04T14:08:04.488Z 13 260-279 166 165 166 286 285 286 F T F <---
180052024-02-04T14:08:04.488Z 14 280-299 166 162 166 286 282 286 F F F <---
180062024-02-04T14:08:04.488Z 15 300-319 165 165 165 285 285 285 F F F
180072024-02-04T14:08:04.488Z 16 320-339 165 165 165 285 285 285 F F F
180082024-02-04T14:08:04.489Z 17 340-359 156 156 156 269 269 269 F F F
180092024-02-04T14:08:04.489Z 18 360-379 166 160 166 286 280 286 F F F <---
180102024-02-04T14:08:04.489Z 19 380-399 154 154 154 267 267 267 F F F
180112024-02-04T14:08:04.489Z 20 400-419 166 165 166 286 285 286 F F F <---
180122024-02-04T14:08:04.489Z 21 420-439 161 161 161 281 281 281 F F F
180132024-02-04T14:08:04.489Z 22 440-459 162 162 162 282 282 282 F F F
180142024-02-04T14:08:04.489Z 23 460-479 164 164 164 284 284 284 F F F
180152024-02-04T14:08:04.489Z 24 480-499 160 160 160 280 280 280 F F F
180162024-02-04T14:08:04.489Z 25 500-519 166 165 166 286 285 286 F F F <---
180172024-02-04T14:08:04.489Z 26 520-539 165 165 165 285 285 285 F F F
180182024-02-04T14:08:04.489Z 27 540-559 165 165 165 285 285 285 F F F
180192024-02-04T14:08:04.489Z 28 560-579 159 159 159 277 277 277 F F F
180202024-02-04T14:08:04.489Z 29 580-599 164 164 164 283 283 283 F F F
180212024-02-04T14:08:04.489ZMax gen: 166, Max flush: 286
180222024-02-04T14:08:04.489ZError: Difference in extent metadata found!
180232024-02-04T14:08:04.493ZA Difference in extent metadata is expected here
180242024-02-04T14:08:04.493ZOn loop 83
180252024-02-04T14:08:04.493Z
180262024-02-04T14:08:04.493ZVerifying data now
180272024-02-04T14:08:09.769ZNo change after verify
180282024-02-04T14:08:09.769ZLoop: 83 Downstairs dump after verify (and repair):
180292024-02-04T14:08:09.794ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180302024-02-04T14:08:09.795Z 0 000-019 162 162 162 282 282 282 F F F
180312024-02-04T14:08:09.795Z 1 020-039 166 166 166 286 286 286 F F F
180322024-02-04T14:08:09.795Z 2 040-059 166 166 166 286 286 286 F F F
180332024-02-04T14:08:09.795Z 3 060-079 167 167 167 287 287 287 F F F
180342024-02-04T14:08:09.795Z 4 080-099 166 166 166 286 286 286 F F F
180352024-02-04T14:08:09.795Z 5 100-119 166 166 166 286 286 286 F F F
180362024-02-04T14:08:09.795Z 6 120-139 167 167 167 287 287 287 F F F
180372024-02-04T14:08:09.798Z 7 140-159 166 166 166 286 286 286 F F F
180382024-02-04T14:08:09.798Z 8 160-179 162 162 162 282 282 282 F F F
180392024-02-04T14:08:09.799Z 9 180-199 159 159 159 277 277 277 F F F
180402024-02-04T14:08:09.799Z 10 200-219 166 166 166 286 286 286 F F F
180412024-02-04T14:08:09.799Z 11 220-239 166 166 166 286 286 286 F F F
180422024-02-04T14:08:09.799Z 12 240-259 166 166 166 286 286 286 F F F
180432024-02-04T14:08:09.799Z 13 260-279 166 166 166 286 286 286 F F F
180442024-02-04T14:08:09.799Z 14 280-299 166 166 166 286 286 286 F F F
180452024-02-04T14:08:09.799Z 15 300-319 165 165 165 285 285 285 F F F
180462024-02-04T14:08:09.799Z 16 320-339 165 165 165 285 285 285 F F F
180472024-02-04T14:08:09.799Z 17 340-359 156 156 156 269 269 269 F F F
180482024-02-04T14:08:09.799Z 18 360-379 166 166 166 286 286 286 F F F
180492024-02-04T14:08:09.799Z 19 380-399 154 154 154 267 267 267 F F F
180502024-02-04T14:08:09.799Z 20 400-419 166 166 166 286 286 286 F F F
180512024-02-04T14:08:09.799Z 21 420-439 161 161 161 281 281 281 F F F
180522024-02-04T14:08:09.799Z 22 440-459 162 162 162 282 282 282 F F F
180532024-02-04T14:08:09.799Z 23 460-479 164 164 164 284 284 284 F F F
180542024-02-04T14:08:09.799Z 24 480-499 160 160 160 280 280 280 F F F
180552024-02-04T14:08:09.799Z 25 500-519 166 166 166 286 286 286 F F F
180562024-02-04T14:08:09.799Z 26 520-539 165 165 165 285 285 285 F F F
180572024-02-04T14:08:09.799Z 27 540-559 165 165 165 285 285 285 F F F
180582024-02-04T14:08:09.799Z 28 560-579 159 159 159 277 277 277 F F F
180592024-02-04T14:08:09.799Z 29 580-599 164 164 164 283 283 283 F F F
180602024-02-04T14:08:09.799ZMax gen: 167, Max flush: 287
180612024-02-04T14:08:09.799Z
180622024-02-04T14:08:09.799ZBegin loop 84
180632024-02-04T14:08:09.800ZDownstairs to restart: 0
180642024-02-04T14:08:09.826Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.85832182Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804}
180652024-02-04T14:08:09.826Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.858794639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804}
180662024-02-04T14:08:09.830Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.858872802Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804}
180672024-02-04T14:08:09.830ZCrucible runtime is spawned
180682024-02-04T14:08:09.831Z{"msg":"Crucible 9d2e7edd-a487-44b0-b8d1-f5c07c153f57 has session id: 73b2742d-8ea7-4195-81ee-530b19b4ab02","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.859155774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180692024-02-04T14:08:09.831Z{"msg":"Upstairs opts: Upstairs UUID: 9d2e7edd-a487-44b0-b8d1-f5c07c153f57, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.859246559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180702024-02-04T14:08:09.831Z{"msg":"Crucible stats registered with UUID: 9d2e7edd-a487-44b0-b8d1-f5c07c153f57","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.859471577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180712024-02-04T14:08:09.831Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.859533732Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804}
180722024-02-04T14:08:09.831Z{"msg":"9d2e7edd-a487-44b0-b8d1-f5c07c153f57 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.859557454Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180732024-02-04T14:08:09.831Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.859743471Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180742024-02-04T14:08:09.831Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.859969003Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180752024-02-04T14:08:09.831Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.860055767Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180762024-02-04T14:08:09.831Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.860273432Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180772024-02-04T14:08:09.831Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.860386222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180782024-02-04T14:08:09.831Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.860430895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180792024-02-04T14:08:09.831Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.86046816Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180802024-02-04T14:08:09.831Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.860507654Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180812024-02-04T14:08:09.831Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.860604119Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180822024-02-04T14:08:09.831Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.860652711Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180832024-02-04T14:08:09.831Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.860717405Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180842024-02-04T14:08:09.831Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.8607645Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180852024-02-04T14:08:09.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.861123144Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180862024-02-04T14:08:09.831Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.861195943Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180872024-02-04T14:08:09.831Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.861240327Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180882024-02-04T14:08:09.831Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.861315602Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180892024-02-04T14:08:09.836Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.861906529Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180902024-02-04T14:08:09.836Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.861998794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180912024-02-04T14:08:09.836Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.862313233Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180922024-02-04T14:08:09.836Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.862384967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180932024-02-04T14:08:09.836Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.862868836Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180942024-02-04T14:08:09.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863268048Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180952024-02-04T14:08:09.836Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863333244Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180962024-02-04T14:08:09.836Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863359909Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180972024-02-04T14:08:09.836Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863400968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180982024-02-04T14:08:09.837Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863425866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
180992024-02-04T14:08:09.837Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863512694Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181002024-02-04T14:08:09.837Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863728668Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181012024-02-04T14:08:09.837Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863792029Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181022024-02-04T14:08:09.837Z{"msg":"[0]R flush_numbers[0..12]: [282, 286, 286, 287, 286, 286, 287, 286, 282, 277, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863818942Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181032024-02-04T14:08:09.837Z{"msg":"[0]R generation[0..12]: [162, 166, 166, 167, 166, 166, 167, 166, 162, 159, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863862566Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181042024-02-04T14:08:09.837Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863888688Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181052024-02-04T14:08:09.837Z{"msg":"[1]R flush_numbers[0..12]: [282, 286, 286, 287, 286, 286, 287, 286, 282, 277, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.863912271Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181062024-02-04T14:08:09.837Z{"msg":"[1]R generation[0..12]: [162, 166, 166, 167, 166, 166, 167, 166, 162, 159, 166, 166]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
181072024-02-04T14:08:09.837Z,"time":"2024-02-04T14:08:08.86397218Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181082024-02-04T14:08:09.837Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864048118Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181092024-02-04T14:08:09.837Z{"msg":"[2]R flush_numbers[0..12]: [282, 286, 286, 287, 286, 286, 287, 286, 282, 277, 286, 286]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864118811Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181102024-02-04T14:08:09.837Z{"msg":"[2]R generation[0..12]: [162, 166, 166, 167, 166, 166, 167, 166, 162, 159, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864146019Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181112024-02-04T14:08:09.837Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864204307Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181122024-02-04T14:08:09.837Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864248942Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181132024-02-04T14:08:09.837Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864279866Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181142024-02-04T14:08:09.837Z{"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.86434687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181152024-02-04T14:08:09.837Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.86441436Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181162024-02-04T14:08:09.837Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864441057Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181172024-02-04T14:08:09.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864461503Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181182024-02-04T14:08:09.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864530644Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181192024-02-04T14:08:09.837Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864566584Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181202024-02-04T14:08:09.837Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181212024-02-04T14:08:09.837Z,"time":"2024-02-04T14:08:08.864642918Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181222024-02-04T14:08:09.838Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864692057Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181232024-02-04T14:08:09.838Z{"msg":"9d2e7edd-a487-44b0-b8d1-f5c07c153f57 is now active with session: 73b2742d-8ea7-4195-81ee-530b19b4ab02","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864729674Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181242024-02-04T14:08:09.838Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864796419Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804,"session_id":"73b2742d-8ea7-4195-81ee-530b19b4ab02"}
181252024-02-04T14:08:09.838Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:08.864865101Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804}
181262024-02-04T14:08:09.838ZLoading write count information from file "/tmp/test_repair_verify.data"
181272024-02-04T14:08:09.838ZRun Repair workload
181282024-02-04T14:08:09.838Z01/30 Write block 368 len 32768 data: 13 12 15 13 9 8 12 13
181292024-02-04T14:08:09.842Z02/30 Read block 262 len 40960
181302024-02-04T14:08:09.842Z03/30 Read block 223 len 40960
181312024-02-04T14:08:09.842Z04/30 Write block 81 len 16384 data: 15 17 15 15
181322024-02-04T14:08:09.842Z05/30 Read block 258 len 24576
181332024-02-04T14:08:09.845Z06/30 Read block 219 len 40960
181342024-02-04T14:08:09.845Z07/30 Write block 43 len 28672 data: 12 12 13 13 12 14 11
181352024-02-04T14:08:09.848Z08/30 Flush
181362024-02-04T14:08:09.852Z09/30 Write block 28 len 24576 data: 9 8 9 9 7 11
181372024-02-04T14:08:09.852Z10/30 Write block 596 len 8192 data: 6 5
181382024-02-04T14:08:09.855Z11/30 Write block 25 len 36864 data: 11 11 8 10 9 10 10 8 12
181392024-02-04T14:08:09.855Z12/30 Write block 266 len 32768 data: 16 13 16 15 15 13 9 10
181402024-02-04T14:08:09.855Z13/30 Flush
181412024-02-04T14:08:09.858Z14/30 Write block 478 len 28672 data: 10 10 8 7 5 8 10
181422024-02-04T14:08:09.861Z15/30 Write block 218 len 16384 data: 8 9 12 13
181432024-02-04T14:08:09.861Z16/30 Read block 31 len 40960
181442024-02-04T14:08:09.864Z17/30 Read block 556 len 12288
181452024-02-04T14:08:09.864Z18/30 Write block 80 len 40960 data: 14 16 18 16 16 14 14 10 9 5
181462024-02-04T14:08:09.864Z19/30 Read block 279 len 4096
181472024-02-04T14:08:09.868Z20/30 Read block 27 len 4096
181482024-02-04T14:08:09.868Z21/30 Write block 401 len 24576 data: 12 14 14 15 17 15
181492024-02-04T14:08:09.868Z22/30 Read block 52 len 8192
181502024-02-04T14:08:09.871Z23/30 Read block 587 len 40960
181512024-02-04T14:08:09.871Z24/30 Write block 71 len 28672 data: 11 11 11 11 10 10 13
181522024-02-04T14:08:09.874Z25/30 Write block 515 len 4096 data: 11
181532024-02-04T14:08:09.874Z26/30 Write block 435 len 40960 data: 16 16 14 11 9 12 11 12 14 15
181542024-02-04T14:08:09.877Z27/30 Write block 443 len 32768 data: 15 16 16 18 17 16 14 13
181552024-02-04T14:08:09.877Z28/30 Read block 569 len 12288
181562024-02-04T14:08:09.884Z29/30 Write block 161 len 32768 data: 11 12 14 12 14 15 16 12
181572024-02-04T14:08:09.884Z30/30 Write block 354 len 40960 data: 13 15 16 11 11 9 10 11 12 13
181582024-02-04T14:08:09.884Z----------------------------------------------------------------
181592024-02-04T14:08:09.884Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
181602024-02-04T14:08:09.884ZStates: Active Active Active
181612024-02-04T14:08:09.884ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181622024-02-04T14:08:09.884Z 1 Acked 1000 Write 8 Sent Done Done false
181632024-02-04T14:08:09.884Z 2 Acked 1001 Read 10 Sent Done Done false
181642024-02-04T14:08:09.884Z 3 Acked 1002 Read 10 Sent Done Done false
181652024-02-04T14:08:09.884Z 4 Acked 1003 Write 4 Sent Done Done false
181662024-02-04T14:08:09.884Z 5 Acked 1004 Read 6 Sent Done Done false
181672024-02-04T14:08:09.884Z 6 Acked 1005 Read 10 Sent Done Done false
181682024-02-04T14:08:09.884Z 7 Acked 1006 Write 7 Sent Done Done false
181692024-02-04T14:08:09.884Z 8 Acked 1007 Flush 0 Sent Done Done false
181702024-02-04T14:08:09.884Z 9 Acked 1008 Write 6 Sent Done Done false
181712024-02-04T14:08:09.884Z 10 Acked 1009 Write 2 Sent Done Done false
181722024-02-04T14:08:09.884Z 11 Acked 1010 Write 9 Sent Done Done false
181732024-02-04T14:08:09.884Z 12 Acked 1011 Write 8 Sent Done Done false
181742024-02-04T14:08:09.884Z 13 Acked 1012 Flush 0 Sent Done Done false
181752024-02-04T14:08:09.884Z 14 Acked 1013 Write 7 Sent Done Done false
181762024-02-04T14:08:09.885Z 15 Acked 1014 Write 4 Sent Done Done false
181772024-02-04T14:08:09.885Z 16 Acked 1015 Read 10 Sent Done Done false
181782024-02-04T14:08:09.885Z 17 Acked 1016 Read 3 Sent Done Done false
181792024-02-04T14:08:09.885Z 18 Acked 1017 Write 10 Sent Done Done false
181802024-02-04T14:08:09.885Z 19 Acked 1018 Read 1 Sent Done Done false
181812024-02-04T14:08:09.885Z 20 Acked 1019 Read 1 Sent Done Done false
181822024-02-04T14:08:09.885Z 21 Acked 1020 Write 6 Sent Done Done false
181832024-02-04T14:08:09.885Z 22 Acked 1021 Read 2 Sent Done Done false
181842024-02-04T14:08:09.885Z 23 Acked 1022 Read 10 Sent Done Done false
181852024-02-04T14:08:09.885Z 24 Acked 1023 Write 7 Sent Done Done false
181862024-02-04T14:08:09.885Z 25 Acked 1024 Write 1 Sent Done Done false
181872024-02-04T14:08:09.885Z 26 Acked 1025 Write 10 Sent Done Done false
181882024-02-04T14:08:09.885Z 27 Acked 1026 Write 8 Sent Done Done false
181892024-02-04T14:08:09.885Z 28 Acked 1027 Read 3 Sent Done Done false
181902024-02-04T14:08:09.885Z 29 Acked 1028 Write 8 Sent Done Done false
181912024-02-04T14:08:09.885Z 30 Acked 1029 Write 10 Sent Sent Sent false
181922024-02-04T14:08:09.885Z STATES DS:0 DS:1 DS:2 TOTAL
181932024-02-04T14:08:09.885Z New 0 0 0 0
181942024-02-04T14:08:09.885Z Sent 30 1 1 32
181952024-02-04T14:08:09.885Z Done 0 29 29 58
181962024-02-04T14:08:09.885Z Skipped 0 0 0 0
181972024-02-04T14:08:09.885Z Error 0 0 0 0
181982024-02-04T14:08:09.885ZLast Flush: 0 1012 1012
181992024-02-04T14:08:09.885ZDownstairs last five completed:
182002024-02-04T14:08:09.886ZUpstairs last five completed: 30 29 28 27 26
182012024-02-04T14:08:09.889Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:08.913725708Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1804}
182022024-02-04T14:08:09.889ZWrote out file "/tmp/test_repair_verify.data"
182032024-02-04T14:08:09.889Z
182042024-02-04T14:08:09.893ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182052024-02-04T14:08:09.922ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182062024-02-04T14:08:09.922Z 0 000-019 162 162 162 282 282 282 F F F
182072024-02-04T14:08:09.922Z 1 020-039 166 168 168 286 289 289 F F F <---
182082024-02-04T14:08:09.922Z 2 040-059 166 168 168 286 288 288 F F F <---
182092024-02-04T14:08:09.922Z 3 060-079 167 167 167 287 287 287 F T T <---
182102024-02-04T14:08:09.922Z 4 080-099 166 168 168 286 288 288 F T T <---
182112024-02-04T14:08:09.925Z 5 100-119 166 166 166 286 286 286 F F F
182122024-02-04T14:08:09.925Z 6 120-139 167 167 167 287 287 287 F F F
182132024-02-04T14:08:09.925Z 7 140-159 166 166 166 286 286 286 F F F
182142024-02-04T14:08:09.925Z 8 160-179 162 162 162 282 282 282 F T T <---
182152024-02-04T14:08:09.925Z 9 180-199 159 159 159 277 277 277 F F F
182162024-02-04T14:08:09.925Z 10 200-219 166 166 166 286 286 286 F T T <---
182172024-02-04T14:08:09.926Z 11 220-239 166 166 166 286 286 286 F T T <---
182182024-02-04T14:08:09.926Z 12 240-259 166 166 166 286 286 286 F F F
182192024-02-04T14:08:09.926Z 13 260-279 166 168 168 286 289 289 F F F <---
182202024-02-04T14:08:09.926Z 14 280-299 166 166 166 286 286 286 F F F
182212024-02-04T14:08:09.926Z 15 300-319 165 165 165 285 285 285 F F F
182222024-02-04T14:08:09.926Z 16 320-339 165 165 165 285 285 285 F F F
182232024-02-04T14:08:09.926Z 17 340-359 156 156 156 269 269 269 F T T <---
182242024-02-04T14:08:09.926Z 18 360-379 166 168 168 286 288 288 F T T <---
182252024-02-04T14:08:09.926Z 19 380-399 154 154 154 267 267 267 F F F
182262024-02-04T14:08:09.926Z 20 400-419 166 166 166 286 286 286 F T T <---
182272024-02-04T14:08:09.926Z 21 420-439 161 161 161 281 281 281 F T T <---
182282024-02-04T14:08:09.926Z 22 440-459 162 162 162 282 282 282 F T T <---
182292024-02-04T14:08:09.926Z 23 460-479 164 164 164 284 284 284 F T T <---
182302024-02-04T14:08:09.926Z 24 480-499 160 160 160 280 280 280 F T T <---
182312024-02-04T14:08:09.926Z 25 500-519 166 166 166 286 286 286 F T T <---
182322024-02-04T14:08:09.926Z 26 520-539 165 165 165 285 285 285 F F F
182332024-02-04T14:08:09.926Z 27 540-559 165 165 165 285 285 285 F F F
182342024-02-04T14:08:09.926Z 28 560-579 159 159 159 277 277 277 F F F
182352024-02-04T14:08:09.926Z 29 580-599 164 168 168 283 289 289 F F F <---
182362024-02-04T14:08:09.926ZMax gen: 168, Max flush: 289
182372024-02-04T14:08:09.926ZError: Difference in extent metadata found!
182382024-02-04T14:08:09.930ZA Difference in extent metadata is expected here
182392024-02-04T14:08:09.930ZOn loop 84
182402024-02-04T14:08:09.930Z
182412024-02-04T14:08:09.933ZVerifying data now
182422024-02-04T14:08:15.353ZNo change after verify
182432024-02-04T14:08:15.353ZLoop: 84 Downstairs dump after verify (and repair):
182442024-02-04T14:08:15.376ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182452024-02-04T14:08:15.376Z 0 000-019 162 162 162 282 282 282 F F F
182462024-02-04T14:08:15.376Z 1 020-039 168 168 168 289 289 289 F F F
182472024-02-04T14:08:15.376Z 2 040-059 168 168 168 288 288 288 F F F
182482024-02-04T14:08:15.376Z 3 060-079 169 169 169 290 290 290 F F F
182492024-02-04T14:08:15.376Z 4 080-099 169 169 169 290 290 290 F F F
182502024-02-04T14:08:15.379Z 5 100-119 166 166 166 286 286 286 F F F
182512024-02-04T14:08:15.379Z 6 120-139 167 167 167 287 287 287 F F F
182522024-02-04T14:08:15.379Z 7 140-159 166 166 166 286 286 286 F F F
182532024-02-04T14:08:15.379Z 8 160-179 169 169 169 290 290 290 F F F
182542024-02-04T14:08:15.379Z 9 180-199 159 159 159 277 277 277 F F F
182552024-02-04T14:08:15.379Z 10 200-219 169 169 169 290 290 290 F F F
182562024-02-04T14:08:15.379Z 11 220-239 169 169 169 290 290 290 F F F
182572024-02-04T14:08:15.379Z 12 240-259 166 166 166 286 286 286 F F F
182582024-02-04T14:08:15.379Z 13 260-279 168 168 168 289 289 289 F F F
182592024-02-04T14:08:15.380Z 14 280-299 166 166 166 286 286 286 F F F
182602024-02-04T14:08:15.380Z 15 300-319 165 165 165 285 285 285 F F F
182612024-02-04T14:08:15.380Z 16 320-339 165 165 165 285 285 285 F F F
182622024-02-04T14:08:15.380Z 17 340-359 169 169 169 290 290 290 F F F
182632024-02-04T14:08:15.380Z 18 360-379 169 169 169 290 290 290 F F F
182642024-02-04T14:08:15.380Z 19 380-399 154 154 154 267 267 267 F F F
182652024-02-04T14:08:15.380Z 20 400-419 169 169 169 290 290 290 F F F
182662024-02-04T14:08:15.380Z 21 420-439 169 169 169 290 290 290 F F F
182672024-02-04T14:08:15.380Z 22 440-459 169 169 169 290 290 290 F F F
182682024-02-04T14:08:15.380Z 23 460-479 169 169 169 290 290 290 F F F
182692024-02-04T14:08:15.380Z 24 480-499 169 169 169 290 290 290 F F F
182702024-02-04T14:08:15.380Z 25 500-519 169 169 169 290 290 290 F F F
182712024-02-04T14:08:15.380Z 26 520-539 165 165 165 285 285 285 F F F
182722024-02-04T14:08:15.380Z 27 540-559 165 165 165 285 285 285 F F F
182732024-02-04T14:08:15.380Z 28 560-579 159 159 159 277 277 277 F F F
182742024-02-04T14:08:15.380Z 29 580-599 168 168 168 289 289 289 F F F
182752024-02-04T14:08:15.380ZMax gen: 169, Max flush: 290
182762024-02-04T14:08:15.383Z
182772024-02-04T14:08:15.383ZBegin loop 85
182782024-02-04T14:08:15.383ZDownstairs to restart: 1
182792024-02-04T14:08:15.413Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.442711761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812}
182802024-02-04T14:08:15.413Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.443154257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812}
182812024-02-04T14:08:15.413Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.443230708Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812}
182822024-02-04T14:08:15.418Z{"msg":"Crucible a23653af-a6eb-498d-9cfa-d9b00b7d56fa has session id: d1088fd0-99cd-4589-b133-42ec89463b81","v":0,"name":"crucible","level":30Crucible runtime is spawned
182832024-02-04T14:08:15.419Z,"time":"2024-02-04T14:08:14.443340109Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182842024-02-04T14:08:15.419Z{"msg":"Upstairs opts: Upstairs UUID: a23653af-a6eb-498d-9cfa-d9b00b7d56fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.443669149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182852024-02-04T14:08:15.419Z{"msg":"Crucible stats registered with UUID: a23653af-a6eb-498d-9cfa-d9b00b7d56fa","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.443769333Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182862024-02-04T14:08:15.419Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.443895359Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812}
182872024-02-04T14:08:15.419Z{"msg":"a23653af-a6eb-498d-9cfa-d9b00b7d56fa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.443938359Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182882024-02-04T14:08:15.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.443976235Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182892024-02-04T14:08:15.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444245774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182902024-02-04T14:08:15.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444293862Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182912024-02-04T14:08:15.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444400831Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182922024-02-04T14:08:15.419Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444446995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182932024-02-04T14:08:15.419Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444530502Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182942024-02-04T14:08:15.419Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444571454Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182952024-02-04T14:08:15.419Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444667961Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182962024-02-04T14:08:15.419Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444711491Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182972024-02-04T14:08:15.419Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444818729Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182982024-02-04T14:08:15.419Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.444945622Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
182992024-02-04T14:08:15.419Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.445071114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183002024-02-04T14:08:15.419Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.445373151Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183012024-02-04T14:08:15.419Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.445431928Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183022024-02-04T14:08:15.419Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.445580991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183032024-02-04T14:08:15.419Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.445647456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183042024-02-04T14:08:15.419Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.445860557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183052024-02-04T14:08:15.419Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.445936031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183062024-02-04T14:08:15.420Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.446342687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183072024-02-04T14:08:15.420Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.446435187Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183082024-02-04T14:08:15.420Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.446905832Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183092024-02-04T14:08:15.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447331423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183102024-02-04T14:08:15.420Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447402393Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183112024-02-04T14:08:15.420Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447428425Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183122024-02-04T14:08:15.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447449607Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183132024-02-04T14:08:15.420Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447504755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183142024-02-04T14:08:15.420Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447528894Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183152024-02-04T14:08:15.420Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447731503Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183162024-02-04T14:08:15.420Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447805505Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183172024-02-04T14:08:15.420Z{"msg":"[0]R flush_numbers[0..12]: [282, 289, 288, 290, 290, 286, 287, 286, 290, 277, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447850475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183182024-02-04T14:08:15.420Z{"msg":"[0]R generation[0..12]: [162, 168, 168, 169, 169, 166, 167, 166, 169, 159, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447908945Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183192024-02-04T14:08:15.420Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.447954799Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183202024-02-04T14:08:15.420Z{"msg":"[1]R flush_numbers[0..12]: [282, 289, 288, 290, 290, 286, 287, 286, 290, 277, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448013388Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812Wait for a query_work_queue command to finish before sending IO
183212024-02-04T14:08:15.420Z,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183222024-02-04T14:08:15.420Z{"msg":"[1]R generation[0..12]: [162, 168, 168, 169, 169, 166, 167, 166, 169, 159, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448061257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183232024-02-04T14:08:15.420Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448121711Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183242024-02-04T14:08:15.420Z{"msg":"[2]R flush_numbers[0..12]: [282, 289, 288, 290, 290, 286, 287, 286, 290, 277, 290, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448166234Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183252024-02-04T14:08:15.420Z{"msg":"[2]R generation[0..12]: [162, 168, 168, 169, 169, 166, 167, 166, 169, 159, 169, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.44822873Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183262024-02-04T14:08:15.420Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448297368Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183272024-02-04T14:08:15.420Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448341066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183282024-02-04T14:08:15.420Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448393411Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183292024-02-04T14:08:15.421Z{"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448434152Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183302024-02-04T14:08:15.421Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448488196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183312024-02-04T14:08:15.421Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448529693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183322024-02-04T14:08:15.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448587141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183332024-02-04T14:08:15.421Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448629303Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183342024-02-04T14:08:15.421Z{"msgRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
183352024-02-04T14:08:15.421Z":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.44869606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183362024-02-04T14:08:15.421Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448758584Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183372024-02-04T14:08:15.421Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448798311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183382024-02-04T14:08:15.421Z{"msg":"a23653af-a6eb-498d-9cfa-d9b00b7d56fa is now active with session: d1088fd0-99cd-4589-b133-42ec89463b81","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448850966Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183392024-02-04T14:08:15.421Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448889236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812,"session_id":"d1088fd0-99cd-4589-b133-42ec89463b81"}
183402024-02-04T14:08:15.421Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:14.448940752Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812}
183412024-02-04T14:08:15.424ZLoading write count information from file "/tmp/test_repair_verify.data"
183422024-02-04T14:08:15.424ZRun Repair workload
183432024-02-04T14:08:15.424Z01/30 Write block 41 len 24576 data: 11 14 13 13 14 14
183442024-02-04T14:08:15.424Z02/30 Write block 36 len 24576 data: 17 16 13 12 11 12
183452024-02-04T14:08:15.424Z03/30 Read block 209 len 4096
183462024-02-04T14:08:15.425Z04/30 Read block 105 len 36864
183472024-02-04T14:08:15.425Z05/30 Read block 87 len 24576
183482024-02-04T14:08:15.427Z06/30 Read block 332 len 32768
183492024-02-04T14:08:15.428Z07/30 Write block 324 len 16384 data: 8 8 10 9
183502024-02-04T14:08:15.430Z08/30 Write block 361 len 16384 data: 12 13 14 12
183512024-02-04T14:08:15.430Z09/30 Write block 191 len 40960 data: 11 11 12 11 10 9 11 13 13 15
183522024-02-04T14:08:15.433Z10/30 Read block 480 len 32768
183532024-02-04T14:08:15.433Z11/30 Read block 336 len 32768
183542024-02-04T14:08:15.436Z12/30 Read block 256 len 16384
183552024-02-04T14:08:15.436Z13/30 Read block 401 len 40960
183562024-02-04T14:08:15.439Z14/30 Read block 206 len 28672
183572024-02-04T14:08:15.439Z15/30 Read block 423 len 20480
183582024-02-04T14:08:15.441Z16/30 Write block 93 len 36864 data: 8 9 11 10 10 9 10 8 9
183592024-02-04T14:08:15.442Z17/30 Read block 72 len 4096
183602024-02-04T14:08:15.444Z18/30 Read block 457 len 28672
183612024-02-04T14:08:15.447Z19/30 Write block 525 len 28672 data: 16 19 18 19 15 13 10
183622024-02-04T14:08:15.447Z20/30 Read block 512 len 36864
183632024-02-04T14:08:15.450Z21/30 Read block 575 len 40960
183642024-02-04T14:08:15.450Z22/30 Write block 32 len 20480 data: 9 13 13 16 18
183652024-02-04T14:08:15.453Z23/30 Write block 478 len 12288 data: 11 11 9
183662024-02-04T14:08:15.453Z24/30 Read block 16 len 40960
183672024-02-04T14:08:15.456Z25/30 Write block 135 len 36864 data: 9 10 9 10 12 17 13 9 10
183682024-02-04T14:08:15.456Z26/30 Read block 130 len 36864
183692024-02-04T14:08:15.459Z27/30 Read block 183 len 40960
183702024-02-04T14:08:15.461Z28/30 Read block 497 len 16384
183712024-02-04T14:08:15.461Z29/30 Read block 502 len 36864
183722024-02-04T14:08:15.465Z30/30 Write block 563 len 4096 data: 14
183732024-02-04T14:08:15.465Z----------------------------------------------------------------
183742024-02-04T14:08:15.465Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183752024-02-04T14:08:15.465ZStates: Active Active Active
183762024-02-04T14:08:15.465ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183772024-02-04T14:08:15.465Z 1 Acked 1000 Write 6 Done Done Done false
183782024-02-04T14:08:15.465Z 2 Acked 1001 Write 6 Done Done Done false
183792024-02-04T14:08:15.465Z 3 Acked 1002 Read 1 Done Sent Done false
183802024-02-04T14:08:15.465Z 4 Acked 1003 Read 9 Done Sent Done false
183812024-02-04T14:08:15.465Z 5 Acked 1004 Read 6 Done Sent Done false
183822024-02-04T14:08:15.465Z 6 Acked 1005 Read 8 Done Sent Done false
183832024-02-04T14:08:15.465Z 7 Acked 1006 Write 4 Done Sent Done false
183842024-02-04T14:08:15.465Z 8 Acked 1007 Write 4 Done Sent Done false
183852024-02-04T14:08:15.465Z 9 Acked 1008 Write 10 Done Sent Done false
183862024-02-04T14:08:15.465Z 10 Acked 1009 Read 8 Done Sent Done false
183872024-02-04T14:08:15.465Z 11 Acked 1010 Read 8 Done Sent Done false
183882024-02-04T14:08:15.465Z 12 Acked 1011 Read 4 Done Sent Done false
183892024-02-04T14:08:15.466Z 13 Acked 1012 Read 10 Done Sent Done false
183902024-02-04T14:08:15.466Z 14 Acked 1013 Read 7 Done Sent Done false
183912024-02-04T14:08:15.466Z 15 Acked 1014 Read 5 Done Sent Done false
183922024-02-04T14:08:15.466Z 16 Acked 1015 Write 9 Done Sent Done false
183932024-02-04T14:08:15.466Z 17 Acked 1016 Read 1 Done Sent Done false
183942024-02-04T14:08:15.466Z 18 Acked 1017 Read 7 Done Sent Done false
183952024-02-04T14:08:15.466Z 19 Acked 1018 Write 7 Done Sent Done false
183962024-02-04T14:08:15.466Z 20 Acked 1019 Read 9 Done Sent Done false
183972024-02-04T14:08:15.466Z 21 Acked 1020 Read 10 Done Sent Done false
183982024-02-04T14:08:15.466Z 22 Acked 1021 Write 5 Done Sent Done false
183992024-02-04T14:08:15.466Z 23 Acked 1022 Write 3 Done Sent Done false
184002024-02-04T14:08:15.466Z 24 Acked 1023 Read 10 Done Sent Done false
184012024-02-04T14:08:15.466Z 25 Acked 1024 Write 9 Done Sent Done false
184022024-02-04T14:08:15.466Z 26 Acked 1025 Read 9 Done Sent Done false
184032024-02-04T14:08:15.466Z 27 Acked 1026 Read 10 Done Sent Done false
184042024-02-04T14:08:15.466Z 28 Acked 1027 Read 4 Done Sent Done false
184052024-02-04T14:08:15.466Z 29 Acked 1028 Read 9 Done Sent Done false
184062024-02-04T14:08:15.466Z 30 Acked 1029 Write 1 Sent Sent Sent false
184072024-02-04T14:08:15.466Z STATES DS:0 DS:1 DS:2 TOTAL
184082024-02-04T14:08:15.466Z New 0 0 0 0
184092024-02-04T14:08:15.466Z Sent 1 28 1 30
184102024-02-04T14:08:15.466Z Done 29 2 29 60
184112024-02-04T14:08:15.466Z Skipped 0 0 0 0
184122024-02-04T14:08:15.466Z Error 0 0 0 0
184132024-02-04T14:08:15.467ZLast Flush: 0 0 0
184142024-02-04T14:08:15.467ZDownstairs last five completed:
184152024-02-04T14:08:15.467ZUpstairs last five completed: 30 29 28 27 26
184162024-02-04T14:08:15.467Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:14.496685975Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1812}
184172024-02-04T14:08:15.467ZWrote out file "/tmp/test_repair_verify.data"
184182024-02-04T14:08:15.470Z
184192024-02-04T14:08:15.479ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
184202024-02-04T14:08:15.504ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184212024-02-04T14:08:15.504Z 0 000-019 162 162 162 282 282 282 F F F
184222024-02-04T14:08:15.504Z 1 020-039 168 168 168 289 289 289 T T T
184232024-02-04T14:08:15.504Z 2 040-059 168 168 168 288 288 288 T T T
184242024-02-04T14:08:15.504Z 3 060-079 169 169 169 290 290 290 F F F
184252024-02-04T14:08:15.504Z 4 080-099 169 169 169 290 290 290 T F T <---
184262024-02-04T14:08:15.505Z 5 100-119 166 166 166 286 286 286 T F T <---
184272024-02-04T14:08:15.505Z 6 120-139 167 167 167 287 287 287 T F T <---
184282024-02-04T14:08:15.508Z 7 140-159 166 166 166 286 286 286 T F T <---
184292024-02-04T14:08:15.508Z 8 160-179 169 169 169 290 290 290 F F F
184302024-02-04T14:08:15.508Z 9 180-199 159 159 159 277 277 277 T F T <---
184312024-02-04T14:08:15.508Z 10 200-219 169 169 169 290 290 290 T F T <---
184322024-02-04T14:08:15.508Z 11 220-239 169 169 169 290 290 290 F F F
184332024-02-04T14:08:15.508Z 12 240-259 166 166 166 286 286 286 F F F
184342024-02-04T14:08:15.508Z 13 260-279 168 168 168 289 289 289 F F F
184352024-02-04T14:08:15.508Z 14 280-299 166 166 166 286 286 286 F F F
184362024-02-04T14:08:15.508Z 15 300-319 165 165 165 285 285 285 F F F
184372024-02-04T14:08:15.508Z 16 320-339 165 165 165 285 285 285 T F T <---
184382024-02-04T14:08:15.508Z 17 340-359 169 169 169 290 290 290 F F F
184392024-02-04T14:08:15.508Z 18 360-379 169 169 169 290 290 290 T F T <---
184402024-02-04T14:08:15.508Z 19 380-399 154 154 154 267 267 267 F F F
184412024-02-04T14:08:15.508Z 20 400-419 169 169 169 290 290 290 F F F
184422024-02-04T14:08:15.508Z 21 420-439 169 169 169 290 290 290 F F F
184432024-02-04T14:08:15.508Z 22 440-459 169 169 169 290 290 290 F F F
184442024-02-04T14:08:15.508Z 23 460-479 169 169 169 290 290 290 T F T <---
184452024-02-04T14:08:15.508Z 24 480-499 169 169 169 290 290 290 T F T <---
184462024-02-04T14:08:15.508Z 25 500-519 169 169 169 290 290 290 F F F
184472024-02-04T14:08:15.509Z 26 520-539 165 165 165 285 285 285 T F T <---
184482024-02-04T14:08:15.509Z 27 540-559 165 165 165 285 285 285 F F F
184492024-02-04T14:08:15.509Z 28 560-579 159 159 159 277 277 277 T F T <---
184502024-02-04T14:08:15.509Z 29 580-599 168 168 168 289 289 289 F F F
184512024-02-04T14:08:15.509ZMax gen: 169, Max flush: 290
184522024-02-04T14:08:15.509ZError: Difference in extent metadata found!
184532024-02-04T14:08:15.512ZA Difference in extent metadata is expected here
184542024-02-04T14:08:15.512ZOn loop 85
184552024-02-04T14:08:15.512Z
184562024-02-04T14:08:15.515ZVerifying data now
184572024-02-04T14:08:21.727ZNo change after verify
184582024-02-04T14:08:21.727ZLoop: 85 Downstairs dump after verify (and repair):
184592024-02-04T14:08:21.750ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184602024-02-04T14:08:21.750Z 0 000-019 162 162 162 282 282 282 F F F
184612024-02-04T14:08:21.750Z 1 020-039 170 170 170 291 291 291 F F F
184622024-02-04T14:08:21.750Z 2 040-059 170 170 170 291 291 291 F F F
184632024-02-04T14:08:21.750Z 3 060-079 169 169 169 290 290 290 F F F
184642024-02-04T14:08:21.750Z 4 080-099 170 170 170 291 291 291 F F F
184652024-02-04T14:08:21.750Z 5 100-119 170 170 170 291 291 291 F F F
184662024-02-04T14:08:21.754Z 6 120-139 170 170 170 291 291 291 F F F
184672024-02-04T14:08:21.754Z 7 140-159 170 170 170 291 291 291 F F F
184682024-02-04T14:08:21.754Z 8 160-179 169 169 169 290 290 290 F F F
184692024-02-04T14:08:21.754Z 9 180-199 170 170 170 291 291 291 F F F
184702024-02-04T14:08:21.754Z 10 200-219 170 170 170 291 291 291 F F F
184712024-02-04T14:08:21.754Z 11 220-239 169 169 169 290 290 290 F F F
184722024-02-04T14:08:21.754Z 12 240-259 166 166 166 286 286 286 F F F
184732024-02-04T14:08:21.754Z 13 260-279 168 168 168 289 289 289 F F F
184742024-02-04T14:08:21.754Z 14 280-299 166 166 166 286 286 286 F F F
184752024-02-04T14:08:21.754Z 15 300-319 165 165 165 285 285 285 F F F
184762024-02-04T14:08:21.754Z 16 320-339 170 170 170 291 291 291 F F F
184772024-02-04T14:08:21.754Z 17 340-359 169 169 169 290 290 290 F F F
184782024-02-04T14:08:21.754Z 18 360-379 170 170 170 291 291 291 F F F
184792024-02-04T14:08:21.754Z 19 380-399 154 154 154 267 267 267 F F F
184802024-02-04T14:08:21.754Z 20 400-419 169 169 169 290 290 290 F F F
184812024-02-04T14:08:21.755Z 21 420-439 169 169 169 290 290 290 F F F
184822024-02-04T14:08:21.755Z 22 440-459 169 169 169 290 290 290 F F F
184832024-02-04T14:08:21.755Z 23 460-479 170 170 170 291 291 291 F F F
184842024-02-04T14:08:21.755Z 24 480-499 170 170 170 291 291 291 F F F
184852024-02-04T14:08:21.755Z 25 500-519 169 169 169 290 290 290 F F F
184862024-02-04T14:08:21.755Z 26 520-539 170 170 170 291 291 291 F F F
184872024-02-04T14:08:21.755Z 27 540-559 165 165 165 285 285 285 F F F
184882024-02-04T14:08:21.755Z 28 560-579 170 170 170 291 291 291 F F F
184892024-02-04T14:08:21.755Z 29 580-599 168 168 168 289 289 289 F F F
184902024-02-04T14:08:21.755ZMax gen: 170, Max flush: 291
184912024-02-04T14:08:21.758Z
184922024-02-04T14:08:21.758ZBegin loop 86
184932024-02-04T14:08:21.758ZDownstairs to restart: 2
184942024-02-04T14:08:21.787Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.819078751Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820}
184952024-02-04T14:08:21.787Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.8195522Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820}
184962024-02-04T14:08:21.791Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.819628077Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820}
184972024-02-04T14:08:21.791Z{"msg":"Crucible 5728a1a8-b10f-457c-8530-839fc450c08b has session id: b4fcc22f-0483-4d9d-98d6-ee6ccec1b816","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.819703897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
184982024-02-04T14:08:21.792Z{"msg":"Upstairs opts: Upstairs UUID: 5728a1a8-b10f-457c-8530-839fc450c08b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.819753296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
184992024-02-04T14:08:21.792Z{"msg":"Crucible stats registered with UUID: 5728a1a8-b10f-457c-8530-839fc450c08b","v":0,"name":"crucible","level":Crucible runtime is spawned
185002024-02-04T14:08:21.792Z30,"time":"2024-02-04T14:08:20.819849605Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185012024-02-04T14:08:21.792Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.81993787Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820}
185022024-02-04T14:08:21.792Z{"msg":"5728a1a8-b10f-457c-8530-839fc450c08b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820151055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185032024-02-04T14:08:21.792Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820263006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185042024-02-04T14:08:21.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820373164Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185052024-02-04T14:08:21.792Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820418024Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185062024-02-04T14:08:21.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820494334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185072024-02-04T14:08:21.792Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820608688Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185082024-02-04T14:08:21.792Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820718582Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185092024-02-04T14:08:21.792Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820761717Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185102024-02-04T14:08:21.792Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820804204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185112024-02-04T14:08:21.792Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820883949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185122024-02-04T14:08:21.792Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820956266Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185132024-02-04T14:08:21.792Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.820998166Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185142024-02-04T14:08:21.792Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.821040898Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185152024-02-04T14:08:21.792Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.822056812Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185162024-02-04T14:08:21.792Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.822155826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185172024-02-04T14:08:21.792Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.822270228Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185182024-02-04T14:08:21.792Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.822451574Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185192024-02-04T14:08:21.807Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.823161211Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185202024-02-04T14:08:21.807Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.823427774Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185212024-02-04T14:08:21.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.82399506Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185222024-02-04T14:08:21.807Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.82408515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185232024-02-04T14:08:21.807Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.824132334Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185242024-02-04T14:08:21.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.824203345Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185252024-02-04T14:08:21.807Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.824242553Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185262024-02-04T14:08:21.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.824416624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185272024-02-04T14:08:21.807Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.82446895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185282024-02-04T14:08:21.807Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.824541374Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185292024-02-04T14:08:21.807Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.825138133Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185302024-02-04T14:08:21.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.825810328Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185312024-02-04T14:08:21.808Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.825890693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185322024-02-04T14:08:21.808Z{"msg":"[0]R flush_numbers[0..12]: [282, 291, 291, 290, 291, 291, 291, 291, 290, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.82593386Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185332024-02-04T14:08:21.808Z{"msg":"[0]R generation[0..12]: [162, 170, 170, 169, 170, 170, 170, 170, 169, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.825996Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185342024-02-04T14:08:21.808Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826063841Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185352024-02-04T14:08:21.808Z{"msg":"[1]R flush_numbers[0..12]: [282, 291, 291, 290, 291, 291, 291, 291, 290, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826133054Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185362024-02-04T14:08:21.808Z{"msg":"[1]R generation[0..12]: [162, 170, 170, 169, 170, 170, 170, 170, 169, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
185372024-02-04T14:08:21.808Z2024-02-04T14:08:20.82616074Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185382024-02-04T14:08:21.808Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826287893Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185392024-02-04T14:08:21.808Z{"msg":"[2]R flush_numbers[0..12]: [282, 291, 291, 290, 291, 291, 291, 291, 290, 291, 291, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826320494Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185402024-02-04T14:08:21.808Z{"msg":"[2]R generation[0..12]: [162, 170, 170, 169, 170, 170, 170, 170, 169, 170, 170, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826376894Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185412024-02-04T14:08:21.808Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826404416Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185422024-02-04T14:08:21.808Z{"msg":"Max found gen is 171","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826475696Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185432024-02-04T14:08:21.808Z{"msg":"Generation requested: 172 >= found:171","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826503296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185442024-02-04T14:08:21.808Z{"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.82657693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185452024-02-04T14:08:21.808Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826603378Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185462024-02-04T14:08:21.808Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826626092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185472024-02-04T14:08:21.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826671193Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185482024-02-04T14:08:21.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826696448Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185492024-02-04T14:08:21.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826753877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185502024-02-04T14:08:21.808Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826825789Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185512024-02-04T14:08:21.808Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826897111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185522024-02-04T14:08:21.808Z{"msg":"5728a1a8-b10f-457c-8530-839fc450c08b is now active with session: b4fcc22f-0483-4d9d-98d6-ee6ccec1b816","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826927754Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185532024-02-04T14:08:21.808Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.826987572Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820,"session_id":"b4fcc22f-0483-4d9d-98d6-ee6ccec1b816"}
185542024-02-04T14:08:21.808Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:20.827023531Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820}
185552024-02-04T14:08:21.809ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
185562024-02-04T14:08:21.809ZLoading write count information from file "/tmp/test_repair_verify.data"
185572024-02-04T14:08:21.809ZRun Repair workload
185582024-02-04T14:08:21.809Z01/30 Write block 193 len 28672 data: 13 12 11 10 12 14 14
185592024-02-04T14:08:21.809Z02/30 Write block 372 len 4096 data: 10
185602024-02-04T14:08:21.809Z03/30 Read block 32 len 8192
185612024-02-04T14:08:21.809Z04/30 Read block 49 len 4096
185622024-02-04T14:08:21.809Z05/30 Write block 274 len 16384 data: 9 9 9 9
185632024-02-04T14:08:21.809Z06/30 Flush
185642024-02-04T14:08:21.809Z07/30 Write block 5 len 4096 data: 12
185652024-02-04T14:08:21.813Z08/30 Write block 268 len 12288 data: 17 16 16
185662024-02-04T14:08:21.813Z09/30 Flush
185672024-02-04T14:08:21.813Z10/30 Write block 6 len 4096 data: 13
185682024-02-04T14:08:21.813Z11/30 Read block 146 len 12288
185692024-02-04T14:08:21.813Z12/30 Read block 531 len 4096
185702024-02-04T14:08:21.813Z13/30 Read block 521 len 40960
185712024-02-04T14:08:21.817Z14/30 Write block 112 len 4096 data: 18
185722024-02-04T14:08:21.817Z15/30 Flush
185732024-02-04T14:08:21.820Z16/30 Write block 475 len 4096 data: 12
185742024-02-04T14:08:21.820Z17/30 Write block 186 len 8192 data: 17 16
185752024-02-04T14:08:21.820Z18/30 Write block 100 len 16384 data: 9 10 9 8
185762024-02-04T14:08:21.823Z19/30 Write block 198 len 24576 data: 15 15 16 15 15 16
185772024-02-04T14:08:21.823Z20/30 Write block 503 len 32768 data: 12 14 11 16 16 17 17 15
185782024-02-04T14:08:21.823Z21/30 Read block 543 len 12288
185792024-02-04T14:08:21.826Z22/30 Write block 313 len 20480 data: 11 11 7 7 8
185802024-02-04T14:08:21.826Z23/30 Read block 356 len 28672
185812024-02-04T14:08:21.829Z24/30 Write block 204 len 24576 data: 18 16 15 14 12 11
185822024-02-04T14:08:21.829Z25/30 Read block 109 len 8192
185832024-02-04T14:08:21.833Z26/30 Read block 547 len 8192
185842024-02-04T14:08:21.833Z27/30 Write block 243 len 12288 data: 12 13 13
185852024-02-04T14:08:21.833Z28/30 Write block 133 len 12288 data: 7 9 10
185862024-02-04T14:08:21.837Z29/30 Write block 568 len 4096 data: 15
185872024-02-04T14:08:21.837Z30/30 Write block 76 len 28672 data: 11 14 13 12 15 17 19
185882024-02-04T14:08:21.837Z----------------------------------------------------------------
185892024-02-04T14:08:21.837Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185902024-02-04T14:08:21.837ZStates: Active Active Active
185912024-02-04T14:08:21.837ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185922024-02-04T14:08:21.837Z 1 Acked 1000 Write 7 Done Done Done false
185932024-02-04T14:08:21.837Z 2 Acked 1001 Write 1 Done Done Done false
185942024-02-04T14:08:21.837Z 3 Acked 1002 Read 2 Done Done Done false
185952024-02-04T14:08:21.837Z 4 Acked 1003 Read 1 Done Done Sent false
185962024-02-04T14:08:21.837Z 5 Acked 1004 Write 4 Done Done Sent false
185972024-02-04T14:08:21.837Z 6 Acked 1005 Flush 0 Done Done Sent false
185982024-02-04T14:08:21.837Z 7 Acked 1006 Write 1 Done Done Sent false
185992024-02-04T14:08:21.837Z 8 Acked 1007 Write 3 Done Done Sent false
186002024-02-04T14:08:21.837Z 9 Acked 1008 Flush 0 Done Done Sent false
186012024-02-04T14:08:21.837Z 10 Acked 1009 Write 1 Done Done Sent false
186022024-02-04T14:08:21.837Z 11 Acked 1010 Read 3 Done Done Sent false
186032024-02-04T14:08:21.837Z 12 Acked 1011 Read 1 Done Done Sent false
186042024-02-04T14:08:21.837Z 13 Acked 1012 Read 10 Done Done Sent false
186052024-02-04T14:08:21.837Z 14 Acked 1013 Write 1 Done Done Sent false
186062024-02-04T14:08:21.837Z 15 Acked 1014 Flush 0 Done Done Sent false
186072024-02-04T14:08:21.837Z 16 Acked 1015 Write 1 Done Done Sent false
186082024-02-04T14:08:21.838Z 17 Acked 1016 Write 2 Done Done Sent false
186092024-02-04T14:08:21.838Z 18 Acked 1017 Write 4 Done Done Sent false
186102024-02-04T14:08:21.838Z 19 Acked 1018 Write 6 Done Done Sent false
186112024-02-04T14:08:21.838Z 20 Acked 1019 Write 8 Done Done Sent false
186122024-02-04T14:08:21.838Z 21 Acked 1020 Read 3 Done Done Sent false
186132024-02-04T14:08:21.838Z 22 Acked 1021 Write 5 Done Done Sent false
186142024-02-04T14:08:21.838Z 23 Acked 1022 Read 7 Done Done Sent false
186152024-02-04T14:08:21.838Z 24 Acked 1023 Write 6 Done Done Sent false
186162024-02-04T14:08:21.838Z 25 Acked 1024 Read 2 Done Done Sent false
186172024-02-04T14:08:21.838Z 26 Acked 1025 Read 2 Done Done Sent false
186182024-02-04T14:08:21.838Z 27 Acked 1026 Write 3 Done Done Sent false
186192024-02-04T14:08:21.838Z 28 Acked 1027 Write 3 Done Done Sent false
186202024-02-04T14:08:21.838Z 29 Acked 1028 Write 1 Done Done Sent false
186212024-02-04T14:08:21.838Z 30 Acked 1029 Write 7 Sent Sent Sent false
186222024-02-04T14:08:21.838Z STATES DS:0 DS:1 DS:2 TOTAL
186232024-02-04T14:08:21.838Z New 0 0 0 0
186242024-02-04T14:08:21.838Z Sent 1 1 27 29
186252024-02-04T14:08:21.838Z Done 29 29 3 61
186262024-02-04T14:08:21.838Z Skipped 0 0 0 0
186272024-02-04T14:08:21.838Z Error 0 0 0 0
186282024-02-04T14:08:21.838ZLast Flush: 1014 1014 0
186292024-02-04T14:08:21.838ZDownstairs last five completed:
186302024-02-04T14:08:21.838ZUpstairs last five completed: 30 29 28 27 26
186312024-02-04T14:08:21.838Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:20.867758784Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1820}
186322024-02-04T14:08:21.838ZWrote out file "/tmp/test_repair_verify.data"
186332024-02-04T14:08:21.842Z
186342024-02-04T14:08:21.848ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
186352024-02-04T14:08:21.879ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186362024-02-04T14:08:21.880Z 0 000-019 172 172 162 294 294 282 F F F <---
186372024-02-04T14:08:21.880Z 1 020-039 170 170 170 291 291 291 F F F
186382024-02-04T14:08:21.883Z 2 040-059 170 170 170 291 291 291 F F F
186392024-02-04T14:08:21.883Z 3 060-079 169 169 169 290 290 290 T T F <---
186402024-02-04T14:08:21.883Z 4 080-099 170 170 170 291 291 291 T T F <---
186412024-02-04T14:08:21.883Z 5 100-119 172 172 170 294 294 291 T T F <---
186422024-02-04T14:08:21.883Z 6 120-139 170 170 170 291 291 291 T T F <---
186432024-02-04T14:08:21.883Z 7 140-159 170 170 170 291 291 291 F F F
186442024-02-04T14:08:21.883Z 8 160-179 169 169 169 290 290 290 F F F
186452024-02-04T14:08:21.883Z 9 180-199 172 172 170 292 292 291 T T T <---
186462024-02-04T14:08:21.883Z 10 200-219 170 170 170 291 291 291 T T F <---
186472024-02-04T14:08:21.883Z 11 220-239 169 169 169 290 290 290 F F F
186482024-02-04T14:08:21.883Z 12 240-259 166 166 166 286 286 286 T T F <---
186492024-02-04T14:08:21.883Z 13 260-279 172 172 168 293 293 289 F F F <---
186502024-02-04T14:08:21.883Z 14 280-299 166 166 166 286 286 286 F F F
186512024-02-04T14:08:21.883Z 15 300-319 165 165 165 285 285 285 T T F <---
186522024-02-04T14:08:21.883Z 16 320-339 170 170 170 291 291 291 F F F
186532024-02-04T14:08:21.884Z 17 340-359 169 169 169 290 290 290 F F F
186542024-02-04T14:08:21.884Z 18 360-379 172 172 170 292 292 291 F F T <---
186552024-02-04T14:08:21.884Z 19 380-399 154 154 154 267 267 267 F F F
186562024-02-04T14:08:21.884Z 20 400-419 169 169 169 290 290 290 F F F
186572024-02-04T14:08:21.884Z 21 420-439 169 169 169 290 290 290 F F F
186582024-02-04T14:08:21.884Z 22 440-459 169 169 169 290 290 290 F F F
186592024-02-04T14:08:21.884Z 23 460-479 170 170 170 291 291 291 T T F <---
186602024-02-04T14:08:21.884Z 24 480-499 170 170 170 291 291 291 F F F
186612024-02-04T14:08:21.884Z 25 500-519 169 169 169 290 290 290 T T F <---
186622024-02-04T14:08:21.884Z 26 520-539 170 170 170 291 291 291 F F F
186632024-02-04T14:08:21.884Z 27 540-559 165 165 165 285 285 285 F F F
186642024-02-04T14:08:21.884Z 28 560-579 170 170 170 291 291 291 T T F <---
186652024-02-04T14:08:21.884Z 29 580-599 168 168 168 289 289 289 F F F
186662024-02-04T14:08:21.884ZMax gen: 172, Max flush: 294
186672024-02-04T14:08:21.884ZError: Difference in extent metadata found!
186682024-02-04T14:08:21.884ZA Difference in extent metadata is expected here
186692024-02-04T14:08:21.884ZOn loop 86
186702024-02-04T14:08:21.884Z
186712024-02-04T14:08:21.890ZVerifying data now
186722024-02-04T14:08:28.081ZNo change after verify
186732024-02-04T14:08:28.081ZLoop: 86 Downstairs dump after verify (and repair):
186742024-02-04T14:08:28.104ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186752024-02-04T14:08:28.104Z 0 000-019 172 172 172 294 294 294 F F F
186762024-02-04T14:08:28.104Z 1 020-039 170 170 170 291 291 291 F F F
186772024-02-04T14:08:28.104Z 2 040-059 170 170 170 291 291 291 F F F
186782024-02-04T14:08:28.104Z 3 060-079 173 173 173 295 295 295 F F F
186792024-02-04T14:08:28.104Z 4 080-099 173 173 173 295 295 295 F F F
186802024-02-04T14:08:28.107Z 5 100-119 173 173 173 295 295 295 F F F
186812024-02-04T14:08:28.107Z 6 120-139 173 173 173 295 295 295 F F F
186822024-02-04T14:08:28.107Z 7 140-159 170 170 170 291 291 291 F F F
186832024-02-04T14:08:28.107Z 8 160-179 169 169 169 290 290 290 F F F
186842024-02-04T14:08:28.107Z 9 180-199 173 173 173 295 295 295 F F F
186852024-02-04T14:08:28.107Z 10 200-219 173 173 173 295 295 295 F F F
186862024-02-04T14:08:28.107Z 11 220-239 169 169 169 290 290 290 F F F
186872024-02-04T14:08:28.107Z 12 240-259 173 173 173 295 295 295 F F F
186882024-02-04T14:08:28.107Z 13 260-279 172 172 172 293 293 293 F F F
186892024-02-04T14:08:28.107Z 14 280-299 166 166 166 286 286 286 F F F
186902024-02-04T14:08:28.107Z 15 300-319 173 173 173 295 295 295 F F F
186912024-02-04T14:08:28.107Z 16 320-339 170 170 170 291 291 291 F F F
186922024-02-04T14:08:28.107Z 17 340-359 169 169 169 290 290 290 F F F
186932024-02-04T14:08:28.108Z 18 360-379 172 172 172 292 292 292 F F F
186942024-02-04T14:08:28.108Z 19 380-399 154 154 154 267 267 267 F F F
186952024-02-04T14:08:28.108Z 20 400-419 169 169 169 290 290 290 F F F
186962024-02-04T14:08:28.108Z 21 420-439 169 169 169 290 290 290 F F F
186972024-02-04T14:08:28.108Z 22 440-459 169 169 169 290 290 290 F F F
186982024-02-04T14:08:28.108Z 23 460-479 173 173 173 295 295 295 F F F
186992024-02-04T14:08:28.108Z 24 480-499 170 170 170 291 291 291 F F F
187002024-02-04T14:08:28.108Z 25 500-519 173 173 173 295 295 295 F F F
187012024-02-04T14:08:28.108Z 26 520-539 170 170 170 291 291 291 F F F
187022024-02-04T14:08:28.108Z 27 540-559 165 165 165 285 285 285 F F F
187032024-02-04T14:08:28.108Z 28 560-579 173 173 173 295 295 295 F F F
187042024-02-04T14:08:28.108Z 29 580-599 168 168 168 289 289 289 F F F
187052024-02-04T14:08:28.108ZMax gen: 173, Max flush: 295
187062024-02-04T14:08:28.111Z
187072024-02-04T14:08:28.111ZBegin loop 87
187082024-02-04T14:08:28.111ZDownstairs to restart: 1
187092024-02-04T14:08:28.138Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.170238291Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828}
187102024-02-04T14:08:28.142Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.170744568Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828}
187112024-02-04T14:08:28.143Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"Crucible runtime is spawned
187122024-02-04T14:08:28.143Zcrucible","level":30,"time":"2024-02-04T14:08:27.170891915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828}
187132024-02-04T14:08:28.143Z{"msg":"Crucible 7908bf70-c8ce-4900-bb0f-b00039ca7068 has session id: d7a49839-5a01-430c-96ef-2aacf0b25e0b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171018663Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187142024-02-04T14:08:28.143Z{"msg":"Upstairs opts: Upstairs UUID: 7908bf70-c8ce-4900-bb0f-b00039ca7068, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.17113545Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187152024-02-04T14:08:28.143Z{"msg":"Crucible stats registered with UUID: 7908bf70-c8ce-4900-bb0f-b00039ca7068","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.17117133Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187162024-02-04T14:08:28.143Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171214491Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828}
187172024-02-04T14:08:28.143Z{"msg":"7908bf70-c8ce-4900-bb0f-b00039ca7068 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171348365Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187182024-02-04T14:08:28.143Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171380787Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187192024-02-04T14:08:28.143Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171405305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187202024-02-04T14:08:28.143Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171427903Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187212024-02-04T14:08:28.143Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171553708Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187222024-02-04T14:08:28.143Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.1718312Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187232024-02-04T14:08:28.143Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171878718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187242024-02-04T14:08:28.143Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171944425Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187252024-02-04T14:08:28.143Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.171987884Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187262024-02-04T14:08:28.143Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.172087444Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187272024-02-04T14:08:28.143Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.172132184Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187282024-02-04T14:08:28.143Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.172211369Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187292024-02-04T14:08:28.143Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.17225468Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187302024-02-04T14:08:28.143Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.172734768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187312024-02-04T14:08:28.143Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.172795642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187322024-02-04T14:08:28.143Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.173037505Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187332024-02-04T14:08:28.144Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.173100253Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187342024-02-04T14:08:28.144Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.173652956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187352024-02-04T14:08:28.144Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.173722726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187362024-02-04T14:08:28.144Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.173772741Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187372024-02-04T14:08:28.148Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.173978645Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187382024-02-04T14:08:28.148Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.174636636Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187392024-02-04T14:08:28.148Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.174697623Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187402024-02-04T14:08:28.148Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.174723583Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187412024-02-04T14:08:28.148Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.174746686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187422024-02-04T14:08:28.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.174785714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187432024-02-04T14:08:28.149Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.174809723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187442024-02-04T14:08:28.149Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.174832208Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187452024-02-04T14:08:28.149Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.175541699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187462024-02-04T14:08:28.149Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.175654754Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187472024-02-04T14:08:28.149Z{"msg":"[0]R flush_numbers[0..12]: [294, 291, 291, 295, 295, 295, 295, 291, 290, 295, 295, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.175732103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187482024-02-04T14:08:28.149Z{"msg":"[0]R generation[0..12]: [172, 170, 170, 173, 173, 173, 173, 170, 169, 173, 173, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.175802025Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187492024-02-04T14:08:28.149Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.17584747Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187502024-02-04T14:08:28.149Z{"msg":"[1]R flush_numbers[0..12]: [294, 291, 291, 295, 295, 295, 295, 291, 290, 295, 295, 290]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
187512024-02-04T14:08:28.149Z,"time":"2024-02-04T14:08:27.175901496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187522024-02-04T14:08:28.149Z{"msg":"[1]R generation[0..12]: [172, 170, 170, 173, 173, 173, 173, 170, 169, 173, 173, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.175946478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187532024-02-04T14:08:28.149Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176005589Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187542024-02-04T14:08:28.149Z{"msg":"[2]R flush_numbers[0..12]: [294, 291, 291, 295, 295, 295, 295, 291, 290, 295, 295, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176051145Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187552024-02-04T14:08:28.149Z{"msg":"[2]R generation[0..12]: [172, 170, 170, 173, 173, 173, 173, 170, 169, 173, 173, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.17608559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187562024-02-04T14:08:28.149Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176110701Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187572024-02-04T14:08:28.149Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176133058Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187582024-02-04T14:08:28.149Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176184111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187592024-02-04T14:08:28.149Z{"msg":"Next flush: 296","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176206938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187602024-02-04T14:08:28.149Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176247832Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187612024-02-04T14:08:28.149Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176271632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187622024-02-04T14:08:28.149Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176318575Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187632024-02-04T14:08:28.149Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176341879Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187642024-02-04T14:08:28.149Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176389156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187652024-02-04T14:08:28.150Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176412662Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187662024-02-04T14:08:28.150Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176452473Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187672024-02-04T14:08:28.150Z{"msg":"7908bf70-c8ce-4900-bb0f-b00039ca7068 is now active with session: d7a49839-5a01-430c-96ef-2aacf0b25e0b","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176475303Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187682024-02-04T14:08:28.150Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176521829Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828,"session_id":"d7a49839-5a01-430c-96ef-2aacf0b25e0b"}
187692024-02-04T14:08:28.150Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:27.176543475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828}
187702024-02-04T14:08:28.150ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187712024-02-04T14:08:28.150ZLoading write count information from file "/tmp/test_repair_verify.data"
187722024-02-04T14:08:28.150ZRun Repair workload
187732024-02-04T14:08:28.150Z01/30 Write block 30 len 8192 data: 11 11
187742024-02-04T14:08:28.156Z02/30 Write block 570 len 4096 data: 14
187752024-02-04T14:08:28.156Z03/30 Write block 505 len 12288 data: 12 17 17
187762024-02-04T14:08:28.156Z04/30 Write block 86 len 20480 data: 15 11 10 6 6
187772024-02-04T14:08:28.156Z05/30 Write block 544 len 36864 data: 14 13 10 9 9 10 10 8 7
187782024-02-04T14:08:28.156Z06/30 Write block 571 len 20480 data: 12 12 11 11 12
187792024-02-04T14:08:28.156Z07/30 Flush
187802024-02-04T14:08:28.159Z08/30 Write block 458 len 40960 data: 18 18 17 16 13 14 14 17 19 17
187812024-02-04T14:08:28.159Z09/30 Write block 91 len 28672 data: 8 9 9 10 12 11 11
187822024-02-04T14:08:28.159Z10/30 Write block 466 len 4096 data: 20
187832024-02-04T14:08:28.162Z11/30 Write block 187 len 8192 data: 17 13
187842024-02-04T14:08:28.162Z12/30 Read block 201 len 24576
187852024-02-04T14:08:28.165Z13/30 Read block 277 len 36864
187862024-02-04T14:08:28.165Z14/30 Read block 170 len 36864
187872024-02-04T14:08:28.168Z15/30 Write block 158 len 8192 data: 9 10
187882024-02-04T14:08:28.168Z16/30 Read block 59 len 40960
187892024-02-04T14:08:28.171Z17/30 Write block 594 len 20480 data: 11 10 7 6 2
187902024-02-04T14:08:28.171Z18/30 Flush
187912024-02-04T14:08:28.174Z19/30 Write block 373 len 16384 data: 9 13 14 9
187922024-02-04T14:08:28.177Z20/30 Flush
187932024-02-04T14:08:28.180Z21/30 Flush
187942024-02-04T14:08:28.180Z22/30 Write block 303 len 40960 data: 8 11 15 14 14 13 12 14 13 10
187952024-02-04T14:08:28.180Z23/30 Read block 464 len 32768
187962024-02-04T14:08:28.183Z24/30 Read block 166 len 8192
187972024-02-04T14:08:28.186Z25/30 Write block 530 len 28672 data: 14 11 10 8 9 8 9
187982024-02-04T14:08:28.186Z26/30 Write block 515 len 28672 data: 12 12 12 12 12 8 10
187992024-02-04T14:08:28.186Z27/30 Read block 554 len 28672
188002024-02-04T14:08:28.189Z28/30 Write block 30 len 16384 data: 12 12 10 14
188012024-02-04T14:08:28.189Z29/30 Read block 247 len 40960
188022024-02-04T14:08:28.193Z30/30 Write block 377 len 12288 data: 7 10 10
188032024-02-04T14:08:28.193Z----------------------------------------------------------------
188042024-02-04T14:08:28.193Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
188052024-02-04T14:08:28.193ZStates: Active Active Active
188062024-02-04T14:08:28.193ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
188072024-02-04T14:08:28.193Z 1 Acked 1000 Write 2 Done Done Done false
188082024-02-04T14:08:28.193Z 2 Acked 1001 Write 1 Done Done Done false
188092024-02-04T14:08:28.193Z 3 Acked 1002 Write 3 Done Done Done false
188102024-02-04T14:08:28.193Z 4 Acked 1003 Write 5 Done Sent Done false
188112024-02-04T14:08:28.193Z 5 Acked 1004 Write 9 Done Sent Done false
188122024-02-04T14:08:28.193Z 6 Acked 1005 Write 5 Done Sent Done false
188132024-02-04T14:08:28.193Z 7 Acked 1006 Flush 0 Done Sent Done false
188142024-02-04T14:08:28.193Z 8 Acked 1007 Write 10 Done Sent Done false
188152024-02-04T14:08:28.193Z 9 Acked 1008 Write 7 Done Sent Done false
188162024-02-04T14:08:28.193Z 10 Acked 1009 Write 1 Done Sent Done false
188172024-02-04T14:08:28.194Z 11 Acked 1010 Write 2 Done Sent Done false
188182024-02-04T14:08:28.194Z 12 Acked 1011 Read 6 Done Sent Done false
188192024-02-04T14:08:28.194Z 13 Acked 1012 Read 9 Done Sent Done false
188202024-02-04T14:08:28.194Z 14 Acked 1013 Read 9 Done Sent Done false
188212024-02-04T14:08:28.194Z 15 Acked 1014 Write 2 Done Sent Done false
188222024-02-04T14:08:28.194Z 16 Acked 1015 Read 10 Done Sent Done false
188232024-02-04T14:08:28.194Z 17 Acked 1016 Write 5 Done Sent Done false
188242024-02-04T14:08:28.194Z 18 Acked 1017 Flush 0 Done Sent Done false
188252024-02-04T14:08:28.194Z 19 Acked 1018 Write 4 Done Sent Done false
188262024-02-04T14:08:28.194Z 20 Acked 1019 Flush 0 Done Sent Done false
188272024-02-04T14:08:28.194Z 21 Acked 1020 Flush 0 Done Sent Done false
188282024-02-04T14:08:28.194Z 22 Acked 1021 Write 10 Done Sent Done false
188292024-02-04T14:08:28.194Z 23 Acked 1022 Read 8 Done Sent Done false
188302024-02-04T14:08:28.194Z 24 Acked 1023 Read 2 Done Sent Done false
188312024-02-04T14:08:28.194Z 25 Acked 1024 Write 7 Done Sent Done false
188322024-02-04T14:08:28.194Z 26 Acked 1025 Write 7 Done Sent Done false
188332024-02-04T14:08:28.194Z 27 Acked 1026 Read 7 Done Sent Done false
188342024-02-04T14:08:28.194Z 28 Acked 1027 Write 4 Done Sent Done false
188352024-02-04T14:08:28.194Z 29 Acked 1028 Read 10 Done Sent Done false
188362024-02-04T14:08:28.194Z 30 Acked 1029 Write 3 Sent Sent Sent false
188372024-02-04T14:08:28.194Z STATES DS:0 DS:1 DS:2 TOTAL
188382024-02-04T14:08:28.194Z New 0 0 0 0
188392024-02-04T14:08:28.194Z Sent 1 27 1 29
188402024-02-04T14:08:28.194Z Done 29 3 29 61
188412024-02-04T14:08:28.194Z Skipped 0 0 0 0
188422024-02-04T14:08:28.195Z Error 0 0 0 0
188432024-02-04T14:08:28.195ZLast Flush: 1020 0 1020
188442024-02-04T14:08:28.195ZDownstairs last five completed:
188452024-02-04T14:08:28.195ZUpstairs last five completed: 30 29 28 27 26
188462024-02-04T14:08:28.195Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:27.225185868Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1828}
188472024-02-04T14:08:28.198ZWrote out file "/tmp/test_repair_verify.data"
188482024-02-04T14:08:28.200Z
188492024-02-04T14:08:28.203ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188502024-02-04T14:08:28.231ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188512024-02-04T14:08:28.231Z 0 000-019 172 172 172 294 294 294 F F F
188522024-02-04T14:08:28.231Z 1 020-039 174 170 174 296 291 296 T T T <---
188532024-02-04T14:08:28.231Z 2 040-059 170 170 170 291 291 291 F F F
188542024-02-04T14:08:28.231Z 3 060-079 173 173 173 295 295 295 F F F
188552024-02-04T14:08:28.231Z 4 080-099 174 173 174 297 295 297 F F F <---
188562024-02-04T14:08:28.231Z 5 100-119 173 173 173 295 295 295 F F F
188572024-02-04T14:08:28.231Z 6 120-139 173 173 173 295 295 295 F F F
188582024-02-04T14:08:28.231Z 7 140-159 174 170 174 297 291 297 F F F <---
188592024-02-04T14:08:28.232Z 8 160-179 169 169 169 290 290 290 F F F
188602024-02-04T14:08:28.232Z 9 180-199 174 173 174 297 295 297 F F F <---
188612024-02-04T14:08:28.232Z 10 200-219 173 173 173 295 295 295 F F F
188622024-02-04T14:08:28.235Z 11 220-239 169 169 169 290 290 290 F F F
188632024-02-04T14:08:28.235Z 12 240-259 173 173 173 295 295 295 F F F
188642024-02-04T14:08:28.235Z 13 260-279 172 172 172 293 293 293 F F F
188652024-02-04T14:08:28.235Z 14 280-299 166 166 166 286 286 286 F F F
188662024-02-04T14:08:28.235Z 15 300-319 173 173 173 295 295 295 T F T <---
188672024-02-04T14:08:28.235Z 16 320-339 170 170 170 291 291 291 F F F
188682024-02-04T14:08:28.235Z 17 340-359 169 169 169 290 290 290 F F F
188692024-02-04T14:08:28.235Z 18 360-379 174 172 174 298 292 298 T F T <---
188702024-02-04T14:08:28.235Z 19 380-399 154 154 154 267 267 267 F F F
188712024-02-04T14:08:28.235Z 20 400-419 169 169 169 290 290 290 F F F
188722024-02-04T14:08:28.235Z 21 420-439 169 169 169 290 290 290 F F F
188732024-02-04T14:08:28.235Z 22 440-459 174 169 174 297 290 297 F F F <---
188742024-02-04T14:08:28.235Z 23 460-479 174 173 174 297 295 297 F F F <---
188752024-02-04T14:08:28.235Z 24 480-499 170 170 170 291 291 291 F F F
188762024-02-04T14:08:28.235Z 25 500-519 174 173 174 296 295 296 T T T <---
188772024-02-04T14:08:28.235Z 26 520-539 170 170 170 291 291 291 T F T <---
188782024-02-04T14:08:28.235Z 27 540-559 174 165 174 296 285 296 F F F <---
188792024-02-04T14:08:28.236Z 28 560-579 174 173 174 296 295 296 F T F <---
188802024-02-04T14:08:28.236Z 29 580-599 174 168 174 297 289 297 F F F <---
188812024-02-04T14:08:28.236ZMax gen: 174, Max flush: 298
188822024-02-04T14:08:28.236ZError: Difference in extent metadata found!
188832024-02-04T14:08:28.236ZA Difference in extent metadata is expected here
188842024-02-04T14:08:28.236ZOn loop 87
188852024-02-04T14:08:28.236Z
188862024-02-04T14:08:28.239ZVerifying data now
188872024-02-04T14:08:34.354ZNo change after verify
188882024-02-04T14:08:34.354ZLoop: 87 Downstairs dump after verify (and repair):
188892024-02-04T14:08:34.377ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188902024-02-04T14:08:34.377Z 0 000-019 172 172 172 294 294 294 F F F
188912024-02-04T14:08:34.377Z 1 020-039 175 175 175 299 299 299 F F F
188922024-02-04T14:08:34.377Z 2 040-059 170 170 170 291 291 291 F F F
188932024-02-04T14:08:34.377Z 3 060-079 173 173 173 295 295 295 F F F
188942024-02-04T14:08:34.377Z 4 080-099 174 174 174 297 297 297 F F F
188952024-02-04T14:08:34.378Z 5 100-119 173 173 173 295 295 295 F F F
188962024-02-04T14:08:34.382Z 6 120-139 173 173 173 295 295 295 F F F
188972024-02-04T14:08:34.382Z 7 140-159 174 174 174 297 297 297 F F F
188982024-02-04T14:08:34.382Z 8 160-179 169 169 169 290 290 290 F F F
188992024-02-04T14:08:34.382Z 9 180-199 174 174 174 297 297 297 F F F
189002024-02-04T14:08:34.382Z 10 200-219 173 173 173 295 295 295 F F F
189012024-02-04T14:08:34.382Z 11 220-239 169 169 169 290 290 290 F F F
189022024-02-04T14:08:34.382Z 12 240-259 173 173 173 295 295 295 F F F
189032024-02-04T14:08:34.382Z 13 260-279 172 172 172 293 293 293 F F F
189042024-02-04T14:08:34.382Z 14 280-299 166 166 166 286 286 286 F F F
189052024-02-04T14:08:34.382Z 15 300-319 175 175 175 299 299 299 F F F
189062024-02-04T14:08:34.383Z 16 320-339 170 170 170 291 291 291 F F F
189072024-02-04T14:08:34.383Z 17 340-359 169 169 169 290 290 290 F F F
189082024-02-04T14:08:34.383Z 18 360-379 175 175 175 299 299 299 F F F
189092024-02-04T14:08:34.383Z 19 380-399 154 154 154 267 267 267 F F F
189102024-02-04T14:08:34.383Z 20 400-419 169 169 169 290 290 290 F F F
189112024-02-04T14:08:34.383Z 21 420-439 169 169 169 290 290 290 F F F
189122024-02-04T14:08:34.383Z 22 440-459 174 174 174 297 297 297 F F F
189132024-02-04T14:08:34.383Z 23 460-479 174 174 174 297 297 297 F F F
189142024-02-04T14:08:34.383Z 24 480-499 170 170 170 291 291 291 F F F
189152024-02-04T14:08:34.383Z 25 500-519 175 175 175 299 299 299 F F F
189162024-02-04T14:08:34.383Z 26 520-539 175 175 175 299 299 299 F F F
189172024-02-04T14:08:34.383Z 27 540-559 174 174 174 296 296 296 F F F
189182024-02-04T14:08:34.384Z 28 560-579 174 174 174 296 296 296 F F F
189192024-02-04T14:08:34.384Z 29 580-599 174 174 174 297 297 297 F F F
189202024-02-04T14:08:34.384ZMax gen: 175, Max flush: 299
189212024-02-04T14:08:34.384Z
189222024-02-04T14:08:34.384ZBegin loop 88
189232024-02-04T14:08:34.384ZDownstairs to restart: 2
189242024-02-04T14:08:34.413Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.443685609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836}
189252024-02-04T14:08:34.413Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444123897Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836}
189262024-02-04T14:08:34.420Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444210828Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836}
189272024-02-04T14:08:34.420Z{"msg":"Crucible d6cf113f-474f-4158-ae04-dca938ea4ab3 has session id: a34ff6bd-cee1-4d24-af06-a813423fef2a","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444283815Z","hostname":"Crucible runtime is spawned
189282024-02-04T14:08:34.420Zip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189292024-02-04T14:08:34.420Z{"msg":"Upstairs opts: Upstairs UUID: d6cf113f-474f-4158-ae04-dca938ea4ab3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444418387Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189302024-02-04T14:08:34.420Z{"msg":"Crucible stats registered with UUID: d6cf113f-474f-4158-ae04-dca938ea4ab3","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444527054Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189312024-02-04T14:08:34.421Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444562439Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836}
189322024-02-04T14:08:34.421Z{"msg":"d6cf113f-474f-4158-ae04-dca938ea4ab3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444582962Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189332024-02-04T14:08:34.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444608339Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189342024-02-04T14:08:34.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444671052Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189352024-02-04T14:08:34.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444701693Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189362024-02-04T14:08:34.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444742485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189372024-02-04T14:08:34.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444777566Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189382024-02-04T14:08:34.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444802375Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189392024-02-04T14:08:34.421Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444829924Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189402024-02-04T14:08:34.421Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444883147Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189412024-02-04T14:08:34.421Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.444922239Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189422024-02-04T14:08:34.421Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.445128697Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189432024-02-04T14:08:34.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.445217853Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189442024-02-04T14:08:34.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.445248201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189452024-02-04T14:08:34.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.446473809Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189462024-02-04T14:08:34.422Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.44661853Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189472024-02-04T14:08:34.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.44670632Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189482024-02-04T14:08:34.422Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.446753052Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189492024-02-04T14:08:34.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.447380955Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189502024-02-04T14:08:34.422Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.447444648Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189512024-02-04T14:08:34.422Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.447562763Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189522024-02-04T14:08:34.422Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.447589768Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189532024-02-04T14:08:34.422Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.448457542Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189542024-02-04T14:08:34.422Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.448519627Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189552024-02-04T14:08:34.422Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.448545709Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189562024-02-04T14:08:34.423Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.448568189Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189572024-02-04T14:08:34.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.448676006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189582024-02-04T14:08:34.423Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.44872812Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189592024-02-04T14:08:34.423Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.448753064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189602024-02-04T14:08:34.428Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449473819Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189612024-02-04T14:08:34.428Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449541471Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189622024-02-04T14:08:34.428Z{"msg":"[0]R flush_numbers[0..12]: [294, 299, 291, 295, 297, 295, 295, 297, 290, 297, 295, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449570059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189632024-02-04T14:08:34.428Z{"msg":"[0]R generation[0..12]: [172, 175, 170, 173, 174, 173, 173, 174, 169, 174, 173, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449594363Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189642024-02-04T14:08:34.428Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449620066Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189652024-02-04T14:08:34.428Z{"msg":"[1]R flush_numbers[0..12]: [294, 299, 291, 295, 297, 295, 295, 297, 290, 297, 295, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449660374Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189662024-02-04T14:08:34.428Z{"msg":"[1]R generation[0..12]: [172, 175, 170, 173, 174, 173, 173, 174, 169, 174, 173, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449765142Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189672024-02-04T14:08:34.428Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
189682024-02-04T14:08:34.429Z,"time":"2024-02-04T14:08:33.449792795Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189692024-02-04T14:08:34.429Z{"msg":"[2]R flush_numbers[0..12]: [294, 299, 291, 295, 297, 295, 295, 297, 290, 297, 295, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449818786Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189702024-02-04T14:08:34.429Z{"msg":"[2]R generation[0..12]: [172, 175, 170, 173, 174, 173, 173, 174, 169, 174, 173, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449842388Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189712024-02-04T14:08:34.429Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449866091Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189722024-02-04T14:08:34.429Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449914901Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189732024-02-04T14:08:34.429Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449938433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189742024-02-04T14:08:34.429Z{"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.449986078Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189752024-02-04T14:08:34.429Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450009697Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189762024-02-04T14:08:34.429Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450032195Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189772024-02-04T14:08:34.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450052261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189782024-02-04T14:08:34.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450111995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189792024-02-04T14:08:34.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450135256Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189802024-02-04T14:08:34.429Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450157616Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189812024-02-04T14:08:34.429Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450208138Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189822024-02-04T14:08:34.429Z{"msg":"d6cf113f-474f-4158-ae04-dca938ea4ab3 is now active with session: a34ff6bd-cee1-4d24-af06-a813423fef2a","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450231246Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189832024-02-04T14:08:34.429Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450251606Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836,"session_id":"a34ff6bd-cee1-4d24-af06-a813423fef2a"}
189842024-02-04T14:08:34.429Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:33.450299647Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836}
189852024-02-04T14:08:34.429ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189862024-02-04T14:08:34.429ZLoading write count information from file "/tmp/test_repair_verify.data"
189872024-02-04T14:08:34.429ZRun Repair workload
189882024-02-04T14:08:34.429Z01/30 Write block 596 len 4096 data: 8
189892024-02-04T14:08:34.429Z02/30 Flush
189902024-02-04T14:08:34.433Z03/30 Write block 2 len 16384 data: 7 8 11 13
189912024-02-04T14:08:34.433Z04/30 Write block 400 len 16384 data: 13 13 15 15
189922024-02-04T14:08:34.433Z05/30 Read block 437 len 40960
189932024-02-04T14:08:34.433Z06/30 Write block 253 len 28672 data: 12 14 14 13 11 13 13
189942024-02-04T14:08:34.433Z07/30 Read block 105 len 32768
189952024-02-04T14:08:34.437Z08/30 Read block 411 len 40960
189962024-02-04T14:08:34.437Z09/30 Read block 137 len 4096
189972024-02-04T14:08:34.440Z10/30 Write block 278 len 16384 data: 11 10 10 9
189982024-02-04T14:08:34.440Z11/30 Flush
189992024-02-04T14:08:34.444Z12/30 Write block 297 len 24576 data: 8 10 11 9 7 9
190002024-02-04T14:08:34.444Z13/30 Read block 216 len 28672
190012024-02-04T14:08:34.447Z14/30 Write block 338 len 28672 data: 10 11 13 13 11 11 11
190022024-02-04T14:08:34.447Z15/30 Write block 354 len 12288 data: 14 16 17
190032024-02-04T14:08:34.451Z16/30 Write block 151 len 40960 data: 12 12 11 10 8 7 8 10 11 10
190042024-02-04T14:08:34.451Z17/30 Write block 597 len 8192 data: 7 3
190052024-02-04T14:08:34.451Z18/30 Write block 197 len 24576 data: 13 16 16 17 16 16
190062024-02-04T14:08:34.454Z19/30 Write block 445 len 28672 data: 17 19 18 17 15 14 12
190072024-02-04T14:08:34.455Z20/30 Read block 492 len 20480
190082024-02-04T14:08:34.458Z21/30 Write block 367 len 32768 data: 15 14 13 16 14 11 10 14
190092024-02-04T14:08:34.458Z22/30 Flush
190102024-02-04T14:08:34.461Z23/30 Flush
190112024-02-04T14:08:34.464Z24/30 Write block 204 len 36864 data: 19 17 16 15 13 12 10 12 12
190122024-02-04T14:08:34.464Z25/30 Read block 91 len 32768
190132024-02-04T14:08:34.468Z26/30 Write block 268 len 16384 data: 18 17 17 14
190142024-02-04T14:08:34.468Z27/30 Read block 207 len 12288
190152024-02-04T14:08:34.468Z28/30 Write block 557 len 20480 data: 16 13 15 16 13
190162024-02-04T14:08:34.472Z29/30 Read block 463 len 16384
190172024-02-04T14:08:34.472Z30/30 Read block 91 len 20480
190182024-02-04T14:08:34.472Z----------------------------------------------------------------
190192024-02-04T14:08:34.473Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:28
190202024-02-04T14:08:34.473ZStates: Active Active Active
190212024-02-04T14:08:34.473ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
190222024-02-04T14:08:34.473Z 3 Acked 1002 Write 4 Done Done Sent false
190232024-02-04T14:08:34.473Z 4 Acked 1003 Write 4 Done Done Sent false
190242024-02-04T14:08:34.473Z 5 Acked 1004 Read 10 Done Done Sent false
190252024-02-04T14:08:34.473Z 6 Acked 1005 Write 7 Done Done Sent false
190262024-02-04T14:08:34.473Z 7 Acked 1006 Read 8 Done Done Sent false
190272024-02-04T14:08:34.473Z 8 Acked 1007 Read 10 Done Done Sent false
190282024-02-04T14:08:34.473Z 9 Acked 1008 Read 1 Done Done Sent false
190292024-02-04T14:08:34.473Z 10 Acked 1009 Write 4 Done Done Sent false
190302024-02-04T14:08:34.473Z 11 Acked 1010 Flush 0 Done Done Sent false
190312024-02-04T14:08:34.473Z 12 Acked 1011 Write 6 Done Done Sent false
190322024-02-04T14:08:34.473Z 13 Acked 1012 Read 7 Done Done Sent false
190332024-02-04T14:08:34.473Z 14 Acked 1013 Write 7 Done Done Sent false
190342024-02-04T14:08:34.474Z 15 Acked 1014 Write 3 Done Done Sent false
190352024-02-04T14:08:34.474Z 16 Acked 1015 Write 10 Done Done Sent false
190362024-02-04T14:08:34.474Z 17 Acked 1016 Write 2 Done Done Sent false
190372024-02-04T14:08:34.474Z 18 Acked 1017 Write 6 Done Done Sent false
190382024-02-04T14:08:34.474Z 19 Acked 1018 Write 7 Done Done Sent false
190392024-02-04T14:08:34.474Z 20 Acked 1019 Read 5 Done Done Sent false
190402024-02-04T14:08:34.474Z 21 Acked 1020 Write 8 Done Done Sent false
190412024-02-04T14:08:34.474Z 22 Acked 1021 Flush 0 Done Done Sent false
190422024-02-04T14:08:34.474Z 23 Acked 1022 Flush 0 Done Done Sent false
190432024-02-04T14:08:34.474Z 24 Acked 1023 Write 9 Done Done Sent false
190442024-02-04T14:08:34.474Z 25 Acked 1024 Read 8 Done Done Sent false
190452024-02-04T14:08:34.474Z 26 Acked 1025 Write 4 Done Done Sent false
190462024-02-04T14:08:34.474Z 27 Acked 1026 Read 3 Done Done Sent false
190472024-02-04T14:08:34.474Z 28 Acked 1027 Write 5 Done Done Sent false
190482024-02-04T14:08:34.474Z 29 Acked 1028 Read 4 Done Done Sent false
190492024-02-04T14:08:34.474Z 30 Acked 1029 Read 5 Sent Done Sent false
190502024-02-04T14:08:34.475Z STATES DS:0 DS:1 DS:2 TOTAL
190512024-02-04T14:08:34.475Z New 0 0 0 0
190522024-02-04T14:08:34.475Z Sent 1 0 28 29
190532024-02-04T14:08:34.475Z Done 27 28 0 55
190542024-02-04T14:08:34.475Z Skipped 0 0 0 0
190552024-02-04T14:08:34.475Z Error 0 0 0 0
190562024-02-04T14:08:34.475ZLast Flush: 1022 1022 1001
190572024-02-04T14:08:34.475ZDownstairs last five completed: 1001 1000
190582024-02-04T14:08:34.475ZUpstairs last five completed: 30 29 28 27 26
190592024-02-04T14:08:34.475Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:33.50461663Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1836}
190602024-02-04T14:08:34.482ZWrote out file "/tmp/test_repair_verify.data"
190612024-02-04T14:08:34.482Z
190622024-02-04T14:08:34.485ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190632024-02-04T14:08:34.508ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190642024-02-04T14:08:34.508Z 0 000-019 176 176 172 301 301 294 F F F <---
190652024-02-04T14:08:34.508Z 1 020-039 175 175 175 299 299 299 F F F
190662024-02-04T14:08:34.508Z 2 040-059 170 170 170 291 291 291 F F F
190672024-02-04T14:08:34.508Z 3 060-079 173 173 173 295 295 295 F F F
190682024-02-04T14:08:34.508Z 4 080-099 174 174 174 297 297 297 F F F
190692024-02-04T14:08:34.508Z 5 100-119 173 173 173 295 295 295 F F F
190702024-02-04T14:08:34.508Z 6 120-139 173 173 173 295 295 295 F F F
190712024-02-04T14:08:34.508Z 7 140-159 176 176 174 302 302 297 F F F <---
190722024-02-04T14:08:34.508Z 8 160-179 176 176 169 302 302 290 F F F <---
190732024-02-04T14:08:34.508Z 9 180-199 176 176 174 302 302 297 F F F <---
190742024-02-04T14:08:34.508Z 10 200-219 176 176 173 302 302 295 T T F <---
190752024-02-04T14:08:34.508Z 11 220-239 169 169 169 290 290 290 F F F
190762024-02-04T14:08:34.508Z 12 240-259 176 176 173 301 301 295 F F F <---
190772024-02-04T14:08:34.508Z 13 260-279 176 176 172 301 301 293 T T F <---
190782024-02-04T14:08:34.512Z 14 280-299 176 176 166 302 302 286 F F F <---
190792024-02-04T14:08:34.512Z 15 300-319 176 176 175 302 302 299 F F F <---
190802024-02-04T14:08:34.512Z 16 320-339 176 176 170 302 302 291 F F F <---
190812024-02-04T14:08:34.512Z 17 340-359 176 176 169 302 302 290 F F F <---
190822024-02-04T14:08:34.512Z 18 360-379 176 176 175 302 302 299 F F F <---
190832024-02-04T14:08:34.512Z 19 380-399 154 154 154 267 267 267 F F F
190842024-02-04T14:08:34.513Z 20 400-419 176 176 169 301 301 290 F F F <---
190852024-02-04T14:08:34.513Z 21 420-439 169 169 169 290 290 290 F F F
190862024-02-04T14:08:34.513Z 22 440-459 176 176 174 302 302 297 F F F <---
190872024-02-04T14:08:34.513Z 23 460-479 174 174 174 297 297 297 F F F
190882024-02-04T14:08:34.513Z 24 480-499 170 170 170 291 291 291 F F F
190892024-02-04T14:08:34.513Z 25 500-519 175 175 175 299 299 299 F F F
190902024-02-04T14:08:34.513Z 26 520-539 175 175 175 299 299 299 F F F
190912024-02-04T14:08:34.513Z 27 540-559 174 174 174 296 296 296 T T F <---
190922024-02-04T14:08:34.513Z 28 560-579 174 174 174 296 296 296 T T F <---
190932024-02-04T14:08:34.513Z 29 580-599 176 176 176 302 302 300 F F F <---
190942024-02-04T14:08:34.513ZMax gen: 176, Max flush: 302
190952024-02-04T14:08:34.513ZError: Difference in extent metadata found!
190962024-02-04T14:08:34.517ZA Difference in extent metadata is expected here
190972024-02-04T14:08:34.517ZOn loop 88
190982024-02-04T14:08:34.517Z
190992024-02-04T14:08:34.517ZVerifying data now
191002024-02-04T14:08:39.926ZNo change after verify
191012024-02-04T14:08:39.926ZLoop: 88 Downstairs dump after verify (and repair):
191022024-02-04T14:08:39.949ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191032024-02-04T14:08:39.949Z 0 000-019 176 176 176 301 301 301 F F F
191042024-02-04T14:08:39.949Z 1 020-039 175 175 175 299 299 299 F F F
191052024-02-04T14:08:39.949Z 2 040-059 170 170 170 291 291 291 F F F
191062024-02-04T14:08:39.949Z 3 060-079 173 173 173 295 295 295 F F F
191072024-02-04T14:08:39.949Z 4 080-099 174 174 174 297 297 297 F F F
191082024-02-04T14:08:39.949Z 5 100-119 173 173 173 295 295 295 F F F
191092024-02-04T14:08:39.949Z 6 120-139 173 173 173 295 295 295 F F F
191102024-02-04T14:08:39.949Z 7 140-159 176 176 176 302 302 302 F F F
191112024-02-04T14:08:39.950Z 8 160-179 176 176 176 302 302 302 F F F
191122024-02-04T14:08:39.950Z 9 180-199 176 176 176 302 302 302 F F F
191132024-02-04T14:08:39.952Z 10 200-219 177 177 177 303 303 303 F F F
191142024-02-04T14:08:39.953Z 11 220-239 169 169 169 290 290 290 F F F
191152024-02-04T14:08:39.953Z 12 240-259 176 176 176 301 301 301 F F F
191162024-02-04T14:08:39.953Z 13 260-279 177 177 177 303 303 303 F F F
191172024-02-04T14:08:39.953Z 14 280-299 176 176 176 302 302 302 F F F
191182024-02-04T14:08:39.953Z 15 300-319 176 176 176 302 302 302 F F F
191192024-02-04T14:08:39.953Z 16 320-339 176 176 176 302 302 302 F F F
191202024-02-04T14:08:39.953Z 17 340-359 176 176 176 302 302 302 F F F
191212024-02-04T14:08:39.953Z 18 360-379 176 176 176 302 302 302 F F F
191222024-02-04T14:08:39.953Z 19 380-399 154 154 154 267 267 267 F F F
191232024-02-04T14:08:39.953Z 20 400-419 176 176 176 301 301 301 F F F
191242024-02-04T14:08:39.953Z 21 420-439 169 169 169 290 290 290 F F F
191252024-02-04T14:08:39.953Z 22 440-459 176 176 176 302 302 302 F F F
191262024-02-04T14:08:39.953Z 23 460-479 174 174 174 297 297 297 F F F
191272024-02-04T14:08:39.953Z 24 480-499 170 170 170 291 291 291 F F F
191282024-02-04T14:08:39.953Z 25 500-519 175 175 175 299 299 299 F F F
191292024-02-04T14:08:39.953Z 26 520-539 175 175 175 299 299 299 F F F
191302024-02-04T14:08:39.953Z 27 540-559 177 177 177 303 303 303 F F F
191312024-02-04T14:08:39.953Z 28 560-579 177 177 177 303 303 303 F F F
191322024-02-04T14:08:39.953Z 29 580-599 176 176 176 302 302 302 F F F
191332024-02-04T14:08:39.953ZMax gen: 177, Max flush: 303
191342024-02-04T14:08:39.957Z
191352024-02-04T14:08:39.957ZBegin loop 89
191362024-02-04T14:08:39.957ZDownstairs to restart: 0
191372024-02-04T14:08:39.985Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.017407639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844}
191382024-02-04T14:08:39.985Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018090378Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844}
191392024-02-04T14:08:39.989Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018140805Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844Crucible runtime is spawned
191402024-02-04T14:08:39.990Z}
191412024-02-04T14:08:39.990Z{"msg":"Crucible e9d946d5-2aa7-4e55-9aa2-ee223e3a7efc has session id: 597db18f-127b-4504-a268-999d7d56c037","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018263069Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191422024-02-04T14:08:39.990Z{"msg":"Upstairs opts: Upstairs UUID: e9d946d5-2aa7-4e55-9aa2-ee223e3a7efc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018348309Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191432024-02-04T14:08:39.990Z{"msg":"Crucible stats registered with UUID: e9d946d5-2aa7-4e55-9aa2-ee223e3a7efc","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018406031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191442024-02-04T14:08:39.990Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018430743Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844}
191452024-02-04T14:08:39.990Z{"msg":"e9d946d5-2aa7-4e55-9aa2-ee223e3a7efc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018450433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191462024-02-04T14:08:39.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018471885Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191472024-02-04T14:08:39.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018516637Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191482024-02-04T14:08:39.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018540994Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191492024-02-04T14:08:39.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018582933Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191502024-02-04T14:08:39.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018606868Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191512024-02-04T14:08:39.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018638076Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191522024-02-04T14:08:39.990Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018721188Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191532024-02-04T14:08:39.990Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018768456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191542024-02-04T14:08:39.990Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.018820699Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191552024-02-04T14:08:39.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.019071437Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191562024-02-04T14:08:39.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.019117269Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191572024-02-04T14:08:39.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.019142145Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191582024-02-04T14:08:39.990Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.020633552Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191592024-02-04T14:08:39.990Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.020731411Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191602024-02-04T14:08:39.990Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.020779467Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191612024-02-04T14:08:39.990Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.020889192Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191622024-02-04T14:08:39.995Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.021604868Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191632024-02-04T14:08:39.995Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.021767336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191642024-02-04T14:08:39.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.021843905Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191652024-02-04T14:08:39.995Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.021869026Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191662024-02-04T14:08:39.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.022699648Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191672024-02-04T14:08:39.995Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.022786752Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191682024-02-04T14:08:39.995Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.022892973Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191692024-02-04T14:08:39.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.022918415Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191702024-02-04T14:08:39.995Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.023012032Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191712024-02-04T14:08:39.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.023047633Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191722024-02-04T14:08:39.996Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.023148851Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191732024-02-04T14:08:39.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.023869008Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191742024-02-04T14:08:39.996Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.023937447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191752024-02-04T14:08:39.996Z{"msg":"[0]R flush_numbers[0..12]: [301, 299, 291, 295, 297, 295, 295, 302, 302, 302, 303, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.023966313Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191762024-02-04T14:08:39.996Z{"msg":"[0]R generation[0..12]: [176, 175, 170, 173, 174, 173, 173, 176, 176, 176, 177, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024010049Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191772024-02-04T14:08:39.996Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024035792Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191782024-02-04T14:08:39.996Z{"msg":"[1]R flush_numbers[0..12]: [301, 299, 291, 295, 297, 295, 295, 302, 302, 302, 303, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024060088Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191792024-02-04T14:08:39.996Z{"msg":"[1]R generation[0..12]: [176, 175, 170, 173, 174, 173, 173, 176, 176, 176, 177, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024083756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191802024-02-04T14:08:39.996Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024140053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191812024-02-04T14:08:39.996Z{"msg":"[2]R flush_numbers[0..12]: [301, 299, 291, 295, 297, 295, 295, 302, 302, 302, 303, 290]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024165033Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844Wait for a query_work_queue command to finish before sending IO
191822024-02-04T14:08:39.996Z,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191832024-02-04T14:08:39.996Z{"msg":"[2]R generation[0..12]: [176, 175, 170, 173, 174, 173, 173, 176, 176, 176, 177, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024198246Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191842024-02-04T14:08:39.996Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024227258Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191852024-02-04T14:08:39.996Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024249639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191862024-02-04T14:08:39.996Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024298746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191872024-02-04T14:08:39.996Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024321993Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191882024-02-04T14:08:39.996Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024362953Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191892024-02-04T14:08:39.996Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024387508Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191902024-02-04T14:08:39.996Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024426537Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191912024-02-04T14:08:39.996Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024450011Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191922024-02-04T14:08:39.996Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024491307Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191932024-02-04T14:08:39.996Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024515433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191942024-02-04T14:08:39.996Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.02453661Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191952024-02-04T14:08:39.997Z{"msg":"e9d946d5-2aa7-4e55-9aa2-ee223e3a7efc is now active with session: 597db18f-127b-4504-a268-999d7d56c037","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024594082Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191962024-02-04T14:08:39.997Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.024616462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844,"session_id":"597db18f-127b-4504-a268-999d7d56c037"}
191972024-02-04T14:08:39.997Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:39.02463713Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844}
191982024-02-04T14:08:39.997ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191992024-02-04T14:08:39.997ZLoading write count information from file "/tmp/test_repair_verify.data"
192002024-02-04T14:08:39.997ZRun Repair workload
192012024-02-04T14:08:39.997Z01/30 Write block 63 len 28672 data: 14 11 11 10 14 14 12
192022024-02-04T14:08:40.001Z02/30 Read block 150 len 4096
192032024-02-04T14:08:40.001Z03/30 Read block 569 len 4096
192042024-02-04T14:08:40.001Z04/30 Read block 324 len 28672
192052024-02-04T14:08:40.001Z05/30 Read block 144 len 8192
192062024-02-04T14:08:40.004Z06/30 Write block 440 len 40960 data: 13 12 13 16 17 18 20 19 18 16
192072024-02-04T14:08:40.004Z07/30 Read block 52 len 4096
192082024-02-04T14:08:40.004Z08/30 Read block 491 len 36864
192092024-02-04T14:08:40.007Z09/30 Write block 222 len 16384 data: 10 10 9 12
192102024-02-04T14:08:40.007Z10/30 Read block 494 len 32768
192112024-02-04T14:08:40.010Z11/30 Read block 433 len 24576
192122024-02-04T14:08:40.010Z12/30 Flush
192132024-02-04T14:08:40.013Z13/30 Write block 577 len 28672 data: 14 14 15 17 16 13 14
192142024-02-04T14:08:40.016Z14/30 Read block 520 len 12288
192152024-02-04T14:08:40.016Z15/30 Read block 353 len 40960
192162024-02-04T14:08:40.019Z16/30 Read block 115 len 4096
192172024-02-04T14:08:40.019Z17/30 Read block 538 len 8192
192182024-02-04T14:08:40.025Z18/30 Read block 108 len 12288
192192024-02-04T14:08:40.025Z19/30 Flush
192202024-02-04T14:08:40.028Z20/30 Write block 182 len 36864 data: 18 20 18 19 18 18 14 13 11
192212024-02-04T14:08:40.028Z21/30 Write block 365 len 24576 data: 11 13 16 15 14 17
192222024-02-04T14:08:40.028Z22/30 Read block 486 len 20480
192232024-02-04T14:08:40.031Z23/30 Read block 36 len 24576
192242024-02-04T14:08:40.031Z24/30 Flush
192252024-02-04T14:08:40.034Z25/30 Write block 35 len 32768 data: 17 19 17 14 13 12 13 15
192262024-02-04T14:08:40.034Z26/30 Read block 200 len 40960
192272024-02-04T14:08:40.036Z27/30 Read block 289 len 20480
192282024-02-04T14:08:40.045Z28/30 Write block 406 len 24576 data: 16 15 14 15 14 15
192292024-02-04T14:08:40.045Z29/30 Write block 399 len 12288 data: 11 14 14
192302024-02-04T14:08:40.048Z30/30 Write block 94 len 32768 data: 11 13 12 12 10 11 10 11
192312024-02-04T14:08:40.048Z----------------------------------------------------------------
192322024-02-04T14:08:40.048Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
192332024-02-04T14:08:40.048ZStates: Active Active Active
192342024-02-04T14:08:40.048ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192352024-02-04T14:08:40.048Z 1 Acked 1000 Write 7 Sent Done Done false
192362024-02-04T14:08:40.048Z 2 Acked 1001 Read 1 Sent Done Done false
192372024-02-04T14:08:40.048Z 3 Acked 1002 Read 1 Sent Done Done false
192382024-02-04T14:08:40.048Z 4 Acked 1003 Read 7 Sent Done Done false
192392024-02-04T14:08:40.049Z 5 Acked 1004 Read 2 Sent Done Done false
192402024-02-04T14:08:40.049Z 6 Acked 1005 Write 10 Sent Done Done false
192412024-02-04T14:08:40.049Z 7 Acked 1006 Read 1 Sent Done Done false
192422024-02-04T14:08:40.049Z 8 Acked 1007 Read 9 Sent Done Done false
192432024-02-04T14:08:40.049Z 9 Acked 1008 Write 4 Sent Done Done false
192442024-02-04T14:08:40.049Z 10 Acked 1009 Read 8 Sent Done Done false
192452024-02-04T14:08:40.049Z 11 Acked 1010 Read 6 Sent Done Done false
192462024-02-04T14:08:40.049Z 12 Acked 1011 Flush 0 Sent Done Done false
192472024-02-04T14:08:40.049Z 13 Acked 1012 Write 7 Sent Done Done false
192482024-02-04T14:08:40.049Z 14 Acked 1013 Read 3 Sent Done Done false
192492024-02-04T14:08:40.049Z 15 Acked 1014 Read 10 Sent Done Done false
192502024-02-04T14:08:40.049Z 16 Acked 1015 Read 1 Sent Done Done false
192512024-02-04T14:08:40.049Z 17 Acked 1016 Read 2 Sent Done Done false
192522024-02-04T14:08:40.049Z 18 Acked 1017 Read 3 Sent Done Done false
192532024-02-04T14:08:40.049Z 19 Acked 1018 Flush 0 Sent Done Done false
192542024-02-04T14:08:40.049Z 20 Acked 1019 Write 9 Sent Done Done false
192552024-02-04T14:08:40.049Z 21 Acked 1020 Write 6 Sent Done Done false
192562024-02-04T14:08:40.049Z 22 Acked 1021 Read 5 Sent Done Done false
192572024-02-04T14:08:40.049Z 23 Acked 1022 Read 6 Sent Done Done false
192582024-02-04T14:08:40.049Z 24 Acked 1023 Flush 0 Sent Done Done false
192592024-02-04T14:08:40.049Z 25 Acked 1024 Write 8 Sent Done Done false
192602024-02-04T14:08:40.050Z 26 Acked 1025 Read 10 Sent Done Done false
192612024-02-04T14:08:40.050Z 27 Acked 1026 Read 5 Sent Done Done false
192622024-02-04T14:08:40.050Z 28 Acked 1027 Write 6 Sent Done Done false
192632024-02-04T14:08:40.050Z 29 Acked 1028 Write 3 Sent Done Done false
192642024-02-04T14:08:40.050Z 30 Acked 1029 Write 8 Sent Sent Sent false
192652024-02-04T14:08:40.050Z STATES DS:0 DS:1 DS:2 TOTAL
192662024-02-04T14:08:40.050Z New 0 0 0 0
192672024-02-04T14:08:40.050Z Sent 30 1 1 32
192682024-02-04T14:08:40.050Z Done 0 29 29 58
192692024-02-04T14:08:40.050Z Skipped 0 0 0 0
192702024-02-04T14:08:40.050Z Error 0 0 0 0
192712024-02-04T14:08:40.050ZLast Flush: 0 1023 1023
192722024-02-04T14:08:40.050ZDownstairs last five completed:
192732024-02-04T14:08:40.050ZUpstairs last five completed: 30 29 28 27 26
192742024-02-04T14:08:40.050Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:39.074837679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1844}
192752024-02-04T14:08:40.050ZWrote out file "/tmp/test_repair_verify.data"
192762024-02-04T14:08:40.053Z
192772024-02-04T14:08:40.056ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192782024-02-04T14:08:40.082ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192792024-02-04T14:08:40.082Z 0 000-019 176 176 176 301 301 301 F F F
192802024-02-04T14:08:40.082Z 1 020-039 175 175 175 299 299 299 F T T <---
192812024-02-04T14:08:40.082Z 2 040-059 170 170 170 291 291 291 F T T <---
192822024-02-04T14:08:40.082Z 3 060-079 173 178 178 295 304 304 F F F <---
192832024-02-04T14:08:40.082Z 4 080-099 174 174 174 297 297 297 F T T <---
192842024-02-04T14:08:40.082Z 5 100-119 173 173 173 295 295 295 F T T <---
192852024-02-04T14:08:40.082Z 6 120-139 173 173 173 295 295 295 F F F
192862024-02-04T14:08:40.082Z 7 140-159 176 176 176 302 302 302 F F F
192872024-02-04T14:08:40.082Z 8 160-179 176 176 176 302 302 302 F F F
192882024-02-04T14:08:40.085Z 9 180-199 176 178 178 302 306 306 F F F <---
192892024-02-04T14:08:40.085Z 10 200-219 177 177 177 303 303 303 F F F
192902024-02-04T14:08:40.085Z 11 220-239 169 178 178 290 304 304 F F F <---
192912024-02-04T14:08:40.085Z 12 240-259 176 176 176 301 301 301 F F F
192922024-02-04T14:08:40.085Z 13 260-279 177 177 177 303 303 303 F F F
192932024-02-04T14:08:40.085Z 14 280-299 176 176 176 302 302 302 F F F
192942024-02-04T14:08:40.085Z 15 300-319 176 176 176 302 302 302 F F F
192952024-02-04T14:08:40.085Z 16 320-339 176 176 176 302 302 302 F F F
192962024-02-04T14:08:40.085Z 17 340-359 176 176 176 302 302 302 F F F
192972024-02-04T14:08:40.085Z 18 360-379 176 178 178 302 306 306 F F F <---
192982024-02-04T14:08:40.085Z 19 380-399 154 154 154 267 267 267 F T T <---
192992024-02-04T14:08:40.085Z 20 400-419 176 176 176 301 301 301 F T T <---
193002024-02-04T14:08:40.085Z 21 420-439 169 169 169 290 290 290 F F F
193012024-02-04T14:08:40.085Z 22 440-459 176 178 178 302 304 304 F F F <---
193022024-02-04T14:08:40.085Z 23 460-479 174 174 174 297 297 297 F F F
193032024-02-04T14:08:40.086Z 24 480-499 170 170 170 291 291 291 F F F
193042024-02-04T14:08:40.086Z 25 500-519 175 175 175 299 299 299 F F F
193052024-02-04T14:08:40.086Z 26 520-539 175 175 175 299 299 299 F F F
193062024-02-04T14:08:40.086Z 27 540-559 177 177 177 303 303 303 F F F
193072024-02-04T14:08:40.086Z 28 560-579 177 178 178 303 305 305 F F F <---
193082024-02-04T14:08:40.086Z 29 580-599 176 178 178 302 305 305 F F F <---
193092024-02-04T14:08:40.086ZMax gen: 178, Max flush: 306
193102024-02-04T14:08:40.086ZError: Difference in extent metadata found!
193112024-02-04T14:08:40.089ZA Difference in extent metadata is expected here
193122024-02-04T14:08:40.089ZOn loop 89
193132024-02-04T14:08:40.089Z
193142024-02-04T14:08:40.091ZVerifying data now
193152024-02-04T14:08:45.237ZNo change after verify
193162024-02-04T14:08:45.237ZLoop: 89 Downstairs dump after verify (and repair):
193172024-02-04T14:08:45.260ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193182024-02-04T14:08:45.260Z 0 000-019 176 176 176 301 301 301 F F F
193192024-02-04T14:08:45.260Z 1 020-039 179 179 179 307 307 307 F F F
193202024-02-04T14:08:45.260Z 2 040-059 179 179 179 307 307 307 F F F
193212024-02-04T14:08:45.260Z 3 060-079 178 178 178 304 304 304 F F F
193222024-02-04T14:08:45.260Z 4 080-099 179 179 179 307 307 307 F F F
193232024-02-04T14:08:45.260Z 5 100-119 179 179 179 307 307 307 F F F
193242024-02-04T14:08:45.260Z 6 120-139 173 173 173 295 295 295 F F F
193252024-02-04T14:08:45.260Z 7 140-159 176 176 176 302 302 302 F F F
193262024-02-04T14:08:45.260Z 8 160-179 176 176 176 302 302 302 F F F
193272024-02-04T14:08:45.260Z 9 180-199 178 178 178 306 306 306 F F F
193282024-02-04T14:08:45.260Z 10 200-219 177 177 177 303 303 303 F F F
193292024-02-04T14:08:45.264Z 11 220-239 178 178 178 304 304 304 F F F
193302024-02-04T14:08:45.264Z 12 240-259 176 176 176 301 301 301 F F F
193312024-02-04T14:08:45.264Z 13 260-279 177 177 177 303 303 303 F F F
193322024-02-04T14:08:45.264Z 14 280-299 176 176 176 302 302 302 F F F
193332024-02-04T14:08:45.264Z 15 300-319 176 176 176 302 302 302 F F F
193342024-02-04T14:08:45.264Z 16 320-339 176 176 176 302 302 302 F F F
193352024-02-04T14:08:45.264Z 17 340-359 176 176 176 302 302 302 F F F
193362024-02-04T14:08:45.264Z 18 360-379 178 178 178 306 306 306 F F F
193372024-02-04T14:08:45.264Z 19 380-399 179 179 179 307 307 307 F F F
193382024-02-04T14:08:45.264Z 20 400-419 179 179 179 307 307 307 F F F
193392024-02-04T14:08:45.264Z 21 420-439 169 169 169 290 290 290 F F F
193402024-02-04T14:08:45.264Z 22 440-459 178 178 178 304 304 304 F F F
193412024-02-04T14:08:45.264Z 23 460-479 174 174 174 297 297 297 F F F
193422024-02-04T14:08:45.264Z 24 480-499 170 170 170 291 291 291 F F F
193432024-02-04T14:08:45.264Z 25 500-519 175 175 175 299 299 299 F F F
193442024-02-04T14:08:45.264Z 26 520-539 175 175 175 299 299 299 F F F
193452024-02-04T14:08:45.264Z 27 540-559 177 177 177 303 303 303 F F F
193462024-02-04T14:08:45.264Z 28 560-579 178 178 178 305 305 305 F F F
193472024-02-04T14:08:45.264Z 29 580-599 178 178 178 305 305 305 F F F
193482024-02-04T14:08:45.265ZMax gen: 179, Max flush: 307
193492024-02-04T14:08:45.267Z
193502024-02-04T14:08:45.267ZBegin loop 90
193512024-02-04T14:08:45.267ZDownstairs to restart: 0
193522024-02-04T14:08:45.295Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.327161312Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852}
193532024-02-04T14:08:45.295Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.327600115Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852}
193542024-02-04T14:08:45.295Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.327636409Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852}
193552024-02-04T14:08:45.300Z{"msg":"Crucible 2f796a8b-049c-4971-95e6-ae7cb2bd4265 has session id: 1d5b704b-666d-4958-b390-f64afdeff0d8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.332504032Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193562024-02-04T14:08:45.300Z{"msg":"Upstairs opts: Upstairs UUID: 2f796a8b-049c-4971-95e6-ae7cb2bd4265, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.332571681Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193572024-02-04T14:08:45.300Z{"msg":"Crucible stats registered with UUID: 2f796a8b-049c-4971-95e6-ae7cb2bd4265","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.332603303Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193582024-02-04T14:08:45.304ZCrucible runtime is spawned
193592024-02-04T14:08:45.304Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333176533Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852}
193602024-02-04T14:08:45.304Z{"msg":"2f796a8b-049c-4971-95e6-ae7cb2bd4265 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333341422Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193612024-02-04T14:08:45.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333405772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193622024-02-04T14:08:45.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333433694Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193632024-02-04T14:08:45.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.33345696Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193642024-02-04T14:08:45.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333494841Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193652024-02-04T14:08:45.304Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333537397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193662024-02-04T14:08:45.304Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333610573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193672024-02-04T14:08:45.304Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333718642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193682024-02-04T14:08:45.304Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333816344Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193692024-02-04T14:08:45.304Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.333918846Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193702024-02-04T14:08:45.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.334020755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193712024-02-04T14:08:45.304Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.334073051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193722024-02-04T14:08:45.305Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.334115781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193732024-02-04T14:08:45.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.335038745Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193742024-02-04T14:08:45.305Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.335099557Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193752024-02-04T14:08:45.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.335239225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193762024-02-04T14:08:45.305Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.335289358Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193772024-02-04T14:08:45.305Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.335313576Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193782024-02-04T14:08:45.305Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.335335645Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193792024-02-04T14:08:45.309Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.336156808Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193802024-02-04T14:08:45.309Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.336303301Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193812024-02-04T14:08:45.309Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.336699459Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193822024-02-04T14:08:45.309Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337136873Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193832024-02-04T14:08:45.309Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337208504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193842024-02-04T14:08:45.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337312419Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193852024-02-04T14:08:45.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337534111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193862024-02-04T14:08:45.310Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337593753Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193872024-02-04T14:08:45.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.33769047Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193882024-02-04T14:08:45.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337733201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193892024-02-04T14:08:45.310Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337757552Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193902024-02-04T14:08:45.310Z{"msg":"[0]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 295, 302, 302, 306, 303, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.33778242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193912024-02-04T14:08:45.310Z{"msg":"[0]R generation[0..12]: [176, 179, 179, 178, 179, 179, 173, 176, 176, 178, 177, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337824838Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193922024-02-04T14:08:45.310Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337850504Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193932024-02-04T14:08:45.310Z{"msg":"[1]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 295, 302, 302, 306, 303, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337874821Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193942024-02-04T14:08:45.310Z{"msg":"[1]R generation[0..12]: [176, 179, 179, 178, 179, 179, 173, 176, 176, 178, 177, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337898222Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193952024-02-04T14:08:45.310Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337923391Z","Wait for a query_work_queue command to finish before sending IO
193962024-02-04T14:08:45.310Zhostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193972024-02-04T14:08:45.310Z{"msg":"[2]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 295, 302, 302, 306, 303, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.337959995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193982024-02-04T14:08:45.310Z{"msg":"[2]R generation[0..12]: [176, 179, 179, 178, 179, 179, 173, 176, 176, 178, 177, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.33798567Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
193992024-02-04T14:08:45.310Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338026092Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194002024-02-04T14:08:45.310Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338064372Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194012024-02-04T14:08:45.310Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338089649Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194022024-02-04T14:08:45.310Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.33811135Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194032024-02-04T14:08:45.310Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338133577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194042024-02-04T14:08:45.310Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338156146Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194052024-02-04T14:08:45.310Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338182638Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194062024-02-04T14:08:45.310Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338245055Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194072024-02-04T14:08:45.311Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338278157Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194082024-02-04T14:08:45.311Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338322853Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194092024-02-04T14:08:45.311Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338345733Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194102024-02-04T14:08:45.311Z{"msg":"2f796a8b-049c-4971-95e6-ae7cb2bd4265 is now active with session: 1d5b704b-666d-4958-b390-f64afdeff0d8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.33836803Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194112024-02-04T14:08:45.311Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338421475Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852,"session_id":"1d5b704b-666d-4958-b390-f64afdeff0d8"}
194122024-02-04T14:08:45.311Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:44.338456115Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852}
194132024-02-04T14:08:45.311ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
194142024-02-04T14:08:45.311ZLoading write count information from file "/tmp/test_repair_verify.data"
194152024-02-04T14:08:45.311ZRun Repair workload
194162024-02-04T14:08:45.311Z01/30 Write block 477 len 40960 data: 9 12 12 10 8 6 9 11 13 10
194172024-02-04T14:08:45.315Z02/30 Write block 161 len 8192 data: 12 13
194182024-02-04T14:08:45.315Z03/30 Write block 305 len 28672 data: 16 15 15 14 13 15 14
194192024-02-04T14:08:45.315Z04/30 Read block 321 len 20480
194202024-02-04T14:08:45.315Z05/30 Flush
194212024-02-04T14:08:45.318Z06/30 Write block 254 len 12288 data: 15 15 14
194222024-02-04T14:08:45.321Z07/30 Write block 276 len 12288 data: 10 10 12
194232024-02-04T14:08:45.321Z08/30 Read block 27 len 12288
194242024-02-04T14:08:45.321Z09/30 Read block 43 len 32768
194252024-02-04T14:08:45.324Z10/30 Write block 164 len 36864 data: 13 15 16 17 13 12 13 11 15
194262024-02-04T14:08:45.324Z11/30 Flush
194272024-02-04T14:08:45.327Z12/30 Write block 146 len 8192 data: 10 8
194282024-02-04T14:08:45.333Z13/30 Write block 318 len 8192 data: 10 13
194292024-02-04T14:08:45.334Z14/30 Write block 530 len 16384 data: 15 12 11 9
194302024-02-04T14:08:45.334Z15/30 Read block 416 len 16384
194312024-02-04T14:08:45.337Z16/30 Read block 264 len 12288
194322024-02-04T14:08:45.337Z17/30 Write block 495 len 24576 data: 13 15 13 15 17 17
194332024-02-04T14:08:45.337Z18/30 Read block 541 len 32768
194342024-02-04T14:08:45.337Z19/30 Read block 263 len 12288
194352024-02-04T14:08:45.340Z20/30 Read block 322 len 32768
194362024-02-04T14:08:45.340Z21/30 Read block 32 len 36864
194372024-02-04T14:08:45.343Z22/30 Read block 406 len 36864
194382024-02-04T14:08:45.343Z23/30 Read block 567 len 12288
194392024-02-04T14:08:45.346Z24/30 Write block 292 len 8192 data: 11 8
194402024-02-04T14:08:45.346Z25/30 Write block 290 len 24576 data: 15 9 12 9 6 8
194412024-02-04T14:08:45.346Z26/30 Read block 157 len 8192
194422024-02-04T14:08:45.350Z27/30 Read block 84 len 32768
194432024-02-04T14:08:45.350Z28/30 Read block 456 len 4096
194442024-02-04T14:08:45.353Z29/30 Write block 458 len 24576 data: 19 19 18 17 14 15
194452024-02-04T14:08:45.353Z30/30 Read block 378 len 20480
194462024-02-04T14:08:45.356Z----------------------------------------------------------------
194472024-02-04T14:08:45.356Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
194482024-02-04T14:08:45.356ZStates: Active Active Active
194492024-02-04T14:08:45.356ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194502024-02-04T14:08:45.356Z 1 Acked 1000 Write 10 Done Done Done false
194512024-02-04T14:08:45.356Z 2 Acked 1001 Write 2 Done Done Done false
194522024-02-04T14:08:45.356Z 3 Acked 1002 Write 7 Sent Done Done false
194532024-02-04T14:08:45.356Z 4 Acked 1003 Read 5 Sent Done Done false
194542024-02-04T14:08:45.357Z 5 Acked 1004 Flush 0 Sent Done Done false
194552024-02-04T14:08:45.357Z 6 Acked 1005 Write 3 Sent Done Done false
194562024-02-04T14:08:45.357Z 7 Acked 1006 Write 3 Sent Done Done false
194572024-02-04T14:08:45.357Z 8 Acked 1007 Read 3 Sent Done Done false
194582024-02-04T14:08:45.357Z 9 Acked 1008 Read 8 Sent Done Done false
194592024-02-04T14:08:45.357Z 10 Acked 1009 Write 9 Sent Done Done false
194602024-02-04T14:08:45.357Z 11 Acked 1010 Flush 0 Sent Done Done false
194612024-02-04T14:08:45.357Z 12 Acked 1011 Write 2 Sent Done Done false
194622024-02-04T14:08:45.357Z 13 Acked 1012 Write 2 Sent Done Done false
194632024-02-04T14:08:45.357Z 14 Acked 1013 Write 4 Sent Done Done false
194642024-02-04T14:08:45.357Z 15 Acked 1014 Read 4 Sent Done Done false
194652024-02-04T14:08:45.357Z 16 Acked 1015 Read 3 Sent Done Done false
194662024-02-04T14:08:45.357Z 17 Acked 1016 Write 6 Sent Done Done false
194672024-02-04T14:08:45.357Z 18 Acked 1017 Read 8 Sent Done Done false
194682024-02-04T14:08:45.357Z 19 Acked 1018 Read 3 Sent Done Done false
194692024-02-04T14:08:45.357Z 20 Acked 1019 Read 8 Sent Done Done false
194702024-02-04T14:08:45.357Z 21 Acked 1020 Read 9 Sent Done Done false
194712024-02-04T14:08:45.357Z 22 Acked 1021 Read 9 Sent Done Done false
194722024-02-04T14:08:45.357Z 23 Acked 1022 Read 3 Sent Done Done false
194732024-02-04T14:08:45.357Z 24 Acked 1023 Write 2 Sent Done Done false
194742024-02-04T14:08:45.357Z 25 Acked 1024 Write 6 Sent Done Done false
194752024-02-04T14:08:45.357Z 26 Acked 1025 Read 2 Sent Done Done false
194762024-02-04T14:08:45.357Z 27 Acked 1026 Read 8 Sent Done Done false
194772024-02-04T14:08:45.357Z 28 Acked 1027 Read 1 Sent Done Done false
194782024-02-04T14:08:45.357Z 29 Acked 1028 Write 6 Sent Done Done false
194792024-02-04T14:08:45.358Z 30 Acked 1029 Read 5 Sent Done Done false
194802024-02-04T14:08:45.358Z STATES DS:0 DS:1 DS:2 TOTAL
194812024-02-04T14:08:45.358Z New 0 0 0 0
194822024-02-04T14:08:45.358Z Sent 28 0 0 28
194832024-02-04T14:08:45.358Z Done 2 30 30 62
194842024-02-04T14:08:45.358Z Skipped 0 0 0 0
194852024-02-04T14:08:45.358Z Error 0 0 0 0
194862024-02-04T14:08:45.358ZLast Flush: 0 1010 1010
194872024-02-04T14:08:45.358ZDownstairs last five completed:
194882024-02-04T14:08:45.358ZUpstairs last five completed: 30 29 28 27 26
194892024-02-04T14:08:45.358Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:44.386209615Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1852}
194902024-02-04T14:08:45.358ZWrote out file "/tmp/test_repair_verify.data"
194912024-02-04T14:08:45.362Z
194922024-02-04T14:08:45.367ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194932024-02-04T14:08:45.395ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194942024-02-04T14:08:45.395Z 0 000-019 176 176 176 301 301 301 F F F
194952024-02-04T14:08:45.395Z 1 020-039 179 179 179 307 307 307 F F F
194962024-02-04T14:08:45.395Z 2 040-059 179 179 179 307 307 307 F F F
194972024-02-04T14:08:45.395Z 3 060-079 178 178 178 304 304 304 F F F
194982024-02-04T14:08:45.395Z 4 080-099 179 179 179 307 307 307 F F F
194992024-02-04T14:08:45.395Z 5 100-119 179 179 179 307 307 307 F F F
195002024-02-04T14:08:45.398Z 6 120-139 173 173 173 295 295 295 F F F
195012024-02-04T14:08:45.398Z 7 140-159 176 176 176 302 302 302 F T T <---
195022024-02-04T14:08:45.398Z 8 160-179 176 180 180 302 309 309 T F F <---
195032024-02-04T14:08:45.398Z 9 180-199 178 178 178 306 306 306 F F F
195042024-02-04T14:08:45.398Z 10 200-219 177 177 177 303 303 303 F F F
195052024-02-04T14:08:45.398Z 11 220-239 178 178 178 304 304 304 F F F
195062024-02-04T14:08:45.398Z 12 240-259 176 180 180 301 309 309 F F F <---
195072024-02-04T14:08:45.398Z 13 260-279 177 180 180 303 309 309 F F F <---
195082024-02-04T14:08:45.398Z 14 280-299 176 176 176 302 302 302 F T T <---
195092024-02-04T14:08:45.398Z 15 300-319 176 180 180 302 308 308 F T T <---
195102024-02-04T14:08:45.398Z 16 320-339 176 176 176 302 302 302 F F F
195112024-02-04T14:08:45.398Z 17 340-359 176 176 176 302 302 302 F F F
195122024-02-04T14:08:45.399Z 18 360-379 178 178 178 306 306 306 F F F
195132024-02-04T14:08:45.399Z 19 380-399 179 179 179 307 307 307 F F F
195142024-02-04T14:08:45.399Z 20 400-419 179 179 179 307 307 307 F F F
195152024-02-04T14:08:45.399Z 21 420-439 169 169 169 290 290 290 F F F
195162024-02-04T14:08:45.399Z 22 440-459 178 178 178 304 304 304 F T T <---
195172024-02-04T14:08:45.399Z 23 460-479 174 180 180 297 308 308 T T T <---
195182024-02-04T14:08:45.399Z 24 480-499 170 180 180 291 308 308 T T T <---
195192024-02-04T14:08:45.399Z 25 500-519 175 175 175 299 299 299 F T T <---
195202024-02-04T14:08:45.399Z 26 520-539 175 175 175 299 299 299 F T T <---
195212024-02-04T14:08:45.399Z 27 540-559 177 177 177 303 303 303 F F F
195222024-02-04T14:08:45.399Z 28 560-579 178 178 178 305 305 305 F F F
195232024-02-04T14:08:45.399Z 29 580-599 178 178 178 305 305 305 F F F
195242024-02-04T14:08:45.399ZMax gen: 180, Max flush: 309
195252024-02-04T14:08:45.399ZError: Difference in extent metadata found!
195262024-02-04T14:08:45.402ZA Difference in extent metadata is expected here
195272024-02-04T14:08:45.403ZOn loop 90
195282024-02-04T14:08:45.403Z
195292024-02-04T14:08:45.405ZVerifying data now
195302024-02-04T14:08:51.521ZNo change after verify
195312024-02-04T14:08:51.521ZLoop: 90 Downstairs dump after verify (and repair):
195322024-02-04T14:08:51.531ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195332024-02-04T14:08:51.531Z 0 000-019 176 176 176 301 301 301 F F F
195342024-02-04T14:08:51.531Z 1 020-039 179 179 179 307 307 307 F F F
195352024-02-04T14:08:51.531Z 2 040-059 179 179 179 307 307 307 F F F
195362024-02-04T14:08:51.531Z 3 060-079 178 178 178 304 304 304 F F F
195372024-02-04T14:08:51.531Z 4 080-099 179 179 179 307 307 307 F F F
195382024-02-04T14:08:51.531Z 5 100-119 179 179 179 307 307 307 F F F
195392024-02-04T14:08:51.532Z 6 120-139 173 173 173 295 295 295 F F F
195402024-02-04T14:08:51.532Z 7 140-159 181 181 181 310 310 310 F F F
195412024-02-04T14:08:51.532Z 8 160-179 180 180 180 309 309 309 F F F
195422024-02-04T14:08:51.532Z 9 180-199 178 178 178 306 306 306 F F F
195432024-02-04T14:08:51.532Z 10 200-219 177 177 177 303 303 303 F F F
195442024-02-04T14:08:51.532Z 11 220-239 178 178 178 304 304 304 F F F
195452024-02-04T14:08:51.532Z 12 240-259 180 180 180 309 309 309 F F F
195462024-02-04T14:08:51.532Z 13 260-279 180 180 180 309 309 309 F F F
195472024-02-04T14:08:51.532Z 14 280-299 181 181 181 310 310 310 F F F
195482024-02-04T14:08:51.532Z 15 300-319 181 181 181 310 310 310 F F F
195492024-02-04T14:08:51.532Z 16 320-339 176 176 176 302 302 302 F F F
195502024-02-04T14:08:51.532Z 17 340-359 176 176 176 302 302 302 F F F
195512024-02-04T14:08:51.532Z 18 360-379 178 178 178 306 306 306 F F F
195522024-02-04T14:08:51.532Z 19 380-399 179 179 179 307 307 307 F F F
195532024-02-04T14:08:51.532Z 20 400-419 179 179 179 307 307 307 F F F
195542024-02-04T14:08:51.532Z 21 420-439 169 169 169 290 290 290 F F F
195552024-02-04T14:08:51.533Z 22 440-459 181 181 181 310 310 310 F F F
195562024-02-04T14:08:51.533Z 23 460-479 181 181 181 310 310 310 F F F
195572024-02-04T14:08:51.533Z 24 480-499 181 181 181 310 310 310 F F F
195582024-02-04T14:08:51.533Z 25 500-519 181 181 181 310 310 310 F F F
195592024-02-04T14:08:51.533Z 26 520-539 181 181 181 310 310 310 F F F
195602024-02-04T14:08:51.533Z 27 540-559 177 177 177 303 303 303 F F F
195612024-02-04T14:08:51.533Z 28 560-579 178 178 178 305 305 305 F F F
195622024-02-04T14:08:51.533Z 29 580-599 178 178 178 305 305 305 F F F
195632024-02-04T14:08:51.533ZMax gen: 181, Max flush: 310
195642024-02-04T14:08:51.533Z
195652024-02-04T14:08:51.533ZBegin loop 91
195662024-02-04T14:08:51.533ZDownstairs to restart: 1
195672024-02-04T14:08:51.533Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.491667756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860}
195682024-02-04T14:08:51.533Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"Crucible runtime is spawned
195692024-02-04T14:08:51.534Z1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492475249Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860}
195702024-02-04T14:08:51.534Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492562219Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860}
195712024-02-04T14:08:51.534Z{"msg":"Crucible da957e49-2364-4084-aa32-91b2260432fa has session id: 471d7c8a-e00e-4735-b5d8-09ac52de9ee0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492590691Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195722024-02-04T14:08:51.534Z{"msg":"Upstairs opts: Upstairs UUID: da957e49-2364-4084-aa32-91b2260432fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492620615Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195732024-02-04T14:08:51.534Z{"msg":"Crucible stats registered with UUID: da957e49-2364-4084-aa32-91b2260432fa","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492643162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195742024-02-04T14:08:51.534Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492664413Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860}
195752024-02-04T14:08:51.534Z{"msg":"da957e49-2364-4084-aa32-91b2260432fa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492776124Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195762024-02-04T14:08:51.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492832313Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195772024-02-04T14:08:51.534Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492857563Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195782024-02-04T14:08:51.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492880852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195792024-02-04T14:08:51.534Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.492922785Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195802024-02-04T14:08:51.534Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.493040491Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195812024-02-04T14:08:51.534Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.493084705Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195822024-02-04T14:08:51.534Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.493157534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195832024-02-04T14:08:51.535Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.493202933Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195842024-02-04T14:08:51.535Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.493246103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195852024-02-04T14:08:51.535Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.493426671Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195862024-02-04T14:08:51.535Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.493732398Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195872024-02-04T14:08:51.535Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.493841031Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195882024-02-04T14:08:51.535Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.49464006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195892024-02-04T14:08:51.535Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.494727702Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195902024-02-04T14:08:51.535Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.494764618Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195912024-02-04T14:08:51.535Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.494808634Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195922024-02-04T14:08:51.535Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.495492478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195932024-02-04T14:08:51.535Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.495701985Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195942024-02-04T14:08:51.535Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.495807858Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195952024-02-04T14:08:51.535Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.495909688Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195962024-02-04T14:08:51.536Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.496738047Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195972024-02-04T14:08:51.536Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.496799832Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195982024-02-04T14:08:51.536Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.496825991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
195992024-02-04T14:08:51.536Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.496865969Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196002024-02-04T14:08:51.536Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.496890164Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196012024-02-04T14:08:51.536Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.496912874Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196022024-02-04T14:08:51.536Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.496953087Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196032024-02-04T14:08:51.536Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.497668407Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196042024-02-04T14:08:51.536Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.497722452Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196052024-02-04T14:08:51.536Z{"msg":"[0]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 295, 310, 309, 306, 303, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.497748932Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196062024-02-04T14:08:51.536Z{"msg":"[0]R generation[0..12]: [176, 179, 179, 178, 179, 179, 173, 181, 180, 178, 177, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.497775863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196072024-02-04T14:08:51.536Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.497835559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196082024-02-04T14:08:51.536Z{"msg":"[1]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 295, 310, 309, 306, 303, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.497862657Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196092024-02-04T14:08:51.537Z{"msg":"[1]R generation[0..12]: [176, 179, 179, 178, 179, 179, 173, 181, 180, 178, 177, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.497920492Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196102024-02-04T14:08:51.537Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.497948666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196112024-02-04T14:08:51.537ZWait for a query_work_queue command to finish before sending IO
196122024-02-04T14:08:51.537Z{"msg":"[2]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 295, 310, 309, 306, 303, 304]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498024916Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196132024-02-04T14:08:51.537Z{"msg":"[2]R generation[0..12]: [176, 179, 179, 178, 179, 179, 173, 181, 180, 178, 177, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498059827Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196142024-02-04T14:08:51.537Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498124204Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196152024-02-04T14:08:51.537Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498149462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196162024-02-04T14:08:51.537Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498179681Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196172024-02-04T14:08:51.537Z{"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498239177Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196182024-02-04T14:08:51.537Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.4982649Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196192024-02-04T14:08:51.537Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498320726Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196202024-02-04T14:08:51.537Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498345288Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196212024-02-04T14:08:51.537Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498404285Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196222024-02-04T14:08:51.538Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.49843036Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196232024-02-04T14:08:51.538Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498487255Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196242024-02-04T14:08:51.538Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498512685Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196252024-02-04T14:08:51.538Z{"msg":"da957e49-2364-4084-aa32-91b2260432fa is now active with session: 471d7c8a-e00e-4735-b5d8-09ac52de9ee0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.49853498Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196262024-02-04T14:08:51.538Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
196272024-02-04T14:08:51.538Z,"time":"2024-02-04T14:08:50.498567772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860,"session_id":"471d7c8a-e00e-4735-b5d8-09ac52de9ee0"}
196282024-02-04T14:08:51.538Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:50.498611137Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860}
196292024-02-04T14:08:51.538ZLoading write count information from file "/tmp/test_repair_verify.data"
196302024-02-04T14:08:51.538ZRun Repair workload
196312024-02-04T14:08:51.538Z01/30 Write block 543 len 4096 data: 16
196322024-02-04T14:08:51.544Z02/30 Write block 180 len 4096 data: 17
196332024-02-04T14:08:51.544Z03/30 Read block 394 len 40960
196342024-02-04T14:08:51.544Z04/30 Read block 293 len 4096
196352024-02-04T14:08:51.544Z05/30 Write block 596 len 16384 data: 9 8 4 2
196362024-02-04T14:08:51.544Z06/30 Write block 324 len 40960 data: 9 9 11 10 8 8 9 10 10 11
196372024-02-04T14:08:51.544Z07/30 Write block 121 len 32768 data: 15 14 13 14 16 16 14 11
196382024-02-04T14:08:51.544Z08/30 Read block 509 len 24576
196392024-02-04T14:08:51.544Z09/30 Write block 429 len 36864 data: 8 11 12 12 12 16 17 17 15
196402024-02-04T14:08:51.544Z10/30 Read block 549 len 40960
196412024-02-04T14:08:51.545Z11/30 Write block 238 len 8192 data: 15 15
196422024-02-04T14:08:51.545Z12/30 Write block 477 len 32768 data: 10 13 13 11 9 7 10 12
196432024-02-04T14:08:51.545Z13/30 Read block 444 len 36864
196442024-02-04T14:08:51.545Z14/30 Read block 109 len 16384
196452024-02-04T14:08:51.545Z15/30 Flush
196462024-02-04T14:08:51.545Z16/30 Write block 289 len 40960 data: 16 16 10 13 10 7 9 7 9 11
196472024-02-04T14:08:51.545Z17/30 Write block 271 len 16384 data: 15 10 11 10
196482024-02-04T14:08:51.545Z18/30 Write block 267 len 28672 data: 14 19 18 18 16 11 12
196492024-02-04T14:08:51.545Z19/30 Read block 573 len 28672
196502024-02-04T14:08:51.545Z20/30 Flush
196512024-02-04T14:08:51.545Z21/30 Write block 237 len 40960 data: 19 16 16 18 16 13 13 14 14 15
196522024-02-04T14:08:51.545Z22/30 Flush
196532024-02-04T14:08:51.545Z23/30 Write block 159 len 20480 data: 12 11 13 14 15
196542024-02-04T14:08:51.546Z24/30 Read block 128 len 32768
196552024-02-04T14:08:51.546Z25/30 Read block 439 len 24576
196562024-02-04T14:08:51.546Z26/30 Read block 206 len 32768
196572024-02-04T14:08:51.546Z27/30 Read block 359 len 20480
196582024-02-04T14:08:51.546Z28/30 Read block 129 len 40960
196592024-02-04T14:08:51.546Z29/30 Write block 471 len 8192 data: 11 12
196602024-02-04T14:08:51.546Z30/30 Read block 230 len 28672
196612024-02-04T14:08:51.546Z----------------------------------------------------------------
196622024-02-04T14:08:51.546Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196632024-02-04T14:08:51.546ZStates: Active Active Active
196642024-02-04T14:08:51.546ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196652024-02-04T14:08:51.546Z 1 Acked 1000 Write 1 Done Done Done false
196662024-02-04T14:08:51.546Z 2 Acked 1001 Write 1 Done Done Done false
196672024-02-04T14:08:51.547Z 3 Acked 1002 Read 10 Done Done Done false
196682024-02-04T14:08:51.547Z 4 Acked 1003 Read 1 Done Done Done false
196692024-02-04T14:08:51.547Z 5 Acked 1004 Write 4 Done Sent Done false
196702024-02-04T14:08:51.547Z 6 Acked 1005 Write 10 Done Sent Done false
196712024-02-04T14:08:51.547Z 7 Acked 1006 Write 8 Done Sent Done false
196722024-02-04T14:08:51.547Z 8 Acked 1007 Read 6 Done Sent Done false
196732024-02-04T14:08:51.547Z 9 Acked 1008 Write 9 Done Sent Done false
196742024-02-04T14:08:51.547Z 10 Acked 1009 Read 10 Done Sent Done false
196752024-02-04T14:08:51.547Z 11 Acked 1010 Write 2 Done Sent Done false
196762024-02-04T14:08:51.547Z 12 Acked 1011 Write 8 Done Sent Done false
196772024-02-04T14:08:51.547Z 13 Acked 1012 Read 9 Done Sent Done false
196782024-02-04T14:08:51.547Z 14 Acked 1013 Read 4 Done Sent Done false
196792024-02-04T14:08:51.547Z 15 Acked 1014 Flush 0 Done Sent Done false
196802024-02-04T14:08:51.548Z 16 Acked 1015 Write 10 Done Sent Done false
196812024-02-04T14:08:51.548Z 17 Acked 1016 Write 4 Done Sent Done false
196822024-02-04T14:08:51.548Z 18 Acked 1017 Write 7 Done Sent Done false
196832024-02-04T14:08:51.548Z 19 Acked 1018 Read 7 Done Sent Done false
196842024-02-04T14:08:51.548Z 20 Acked 1019 Flush 0 Done Sent Done false
196852024-02-04T14:08:51.548Z 21 Acked 1020 Write 10 Done Sent Done false
196862024-02-04T14:08:51.548Z 22 Acked 1021 Flush 0 Done Sent Done false
196872024-02-04T14:08:51.548Z 23 Acked 1022 Write 5 Done Sent Done false
196882024-02-04T14:08:51.548Z 24 Acked 1023 Read 8 Done Sent Done false
196892024-02-04T14:08:51.548Z 25 Acked 1024 Read 6 Done Sent Done false
196902024-02-04T14:08:51.548Z 26 Acked 1025 Read 8 Done Sent Done false
196912024-02-04T14:08:51.548Z 27 Acked 1026 Read 5 Done Sent Done false
196922024-02-04T14:08:51.548Z 28 Acked 1027 Read 10 Done Sent Done false
196932024-02-04T14:08:51.549Z 29 Acked 1028 Write 2 Done Sent Done false
196942024-02-04T14:08:51.549Z 30 Acked 1029 Read 7 Done Sent Done false
196952024-02-04T14:08:51.549Z STATES DS:0 DS:1 DS:2 TOTAL
196962024-02-04T14:08:51.549Z New 0 0 0 0
196972024-02-04T14:08:51.549Z Sent 0 26 0 26
196982024-02-04T14:08:51.549Z Done 30 4 30 64
196992024-02-04T14:08:51.549Z Skipped 0 0 0 0
197002024-02-04T14:08:51.549Z Error 0 0 0 0
197012024-02-04T14:08:51.549ZLast Flush: 1021 0 1021
197022024-02-04T14:08:51.549ZDownstairs last five completed:
197032024-02-04T14:08:51.549ZUpstairs last five completed: 30 29 28 27 26
197042024-02-04T14:08:51.549Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:50.552167037Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1860}
197052024-02-04T14:08:51.549ZWrote out file "/tmp/test_repair_verify.data"
197062024-02-04T14:08:51.550Z
197072024-02-04T14:08:51.550ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
197082024-02-04T14:08:51.568ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197092024-02-04T14:08:51.568Z 0 000-019 176 176 176 301 301 301 F F F
197102024-02-04T14:08:51.568Z 1 020-039 179 179 179 307 307 307 F F F
197112024-02-04T14:08:51.568Z 2 040-059 179 179 179 307 307 307 F F F
197122024-02-04T14:08:51.568Z 3 060-079 178 178 178 304 304 304 F F F
197132024-02-04T14:08:51.568Z 4 080-099 179 179 179 307 307 307 F F F
197142024-02-04T14:08:51.573Z 5 100-119 179 179 179 307 307 307 F F F
197152024-02-04T14:08:51.573Z 6 120-139 182 173 182 311 295 311 F F F <---
197162024-02-04T14:08:51.573Z 7 140-159 181 181 181 310 310 310 T F T <---
197172024-02-04T14:08:51.573Z 8 160-179 180 180 180 309 309 309 T F T <---
197182024-02-04T14:08:51.573Z 9 180-199 182 178 182 311 306 311 F T F <---
197192024-02-04T14:08:51.573Z 10 200-219 177 177 177 303 303 303 F F F
197202024-02-04T14:08:51.573Z 11 220-239 182 178 182 313 304 313 F F F <---
197212024-02-04T14:08:51.573Z 12 240-259 182 180 182 313 309 313 F F F <---
197222024-02-04T14:08:51.573Z 13 260-279 182 180 182 312 309 312 F F F <---
197232024-02-04T14:08:51.573Z 14 280-299 182 181 182 312 310 312 F F F <---
197242024-02-04T14:08:51.574Z 15 300-319 181 181 181 310 310 310 F F F
197252024-02-04T14:08:51.574Z 16 320-339 182 176 182 311 302 311 F F F <---
197262024-02-04T14:08:51.574Z 17 340-359 176 176 176 302 302 302 F F F
197272024-02-04T14:08:51.574Z 18 360-379 178 178 178 306 306 306 F F F
197282024-02-04T14:08:51.574Z 19 380-399 179 179 179 307 307 307 F F F
197292024-02-04T14:08:51.574Z 20 400-419 179 179 179 307 307 307 F F F
197302024-02-04T14:08:51.574Z 21 420-439 182 169 182 311 290 311 F F F <---
197312024-02-04T14:08:51.574Z 22 440-459 181 181 181 310 310 310 F F F
197322024-02-04T14:08:51.574Z 23 460-479 182 181 182 311 310 311 T F T <---
197332024-02-04T14:08:51.574Z 24 480-499 182 181 182 311 310 311 F F F <---
197342024-02-04T14:08:51.574Z 25 500-519 181 181 181 310 310 310 F F F
197352024-02-04T14:08:51.574Z 26 520-539 181 181 181 310 310 310 F F F
197362024-02-04T14:08:51.574Z 27 540-559 182 177 182 311 303 311 F T F <---
197372024-02-04T14:08:51.574Z 28 560-579 178 178 178 305 305 305 F F F
197382024-02-04T14:08:51.574Z 29 580-599 182 178 182 311 305 311 F F F <---
197392024-02-04T14:08:51.575ZMax gen: 182, Max flush: 313
197402024-02-04T14:08:51.575ZError: Difference in extent metadata found!
197412024-02-04T14:08:51.575ZA Difference in extent metadata is expected here
197422024-02-04T14:08:51.575ZOn loop 91
197432024-02-04T14:08:51.575Z
197442024-02-04T14:08:51.575ZVerifying data now
197452024-02-04T14:08:56.751ZNo change after verify
197462024-02-04T14:08:56.751ZLoop: 91 Downstairs dump after verify (and repair):
197472024-02-04T14:08:56.774ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197482024-02-04T14:08:56.774Z 0 000-019 176 176 176 301 301 301 F F F
197492024-02-04T14:08:56.774Z 1 020-039 179 179 179 307 307 307 F F F
197502024-02-04T14:08:56.775Z 2 040-059 179 179 179 307 307 307 F F F
197512024-02-04T14:08:56.775Z 3 060-079 178 178 178 304 304 304 F F F
197522024-02-04T14:08:56.775Z 4 080-099 179 179 179 307 307 307 F F F
197532024-02-04T14:08:56.775Z 5 100-119 179 179 179 307 307 307 F F F
197542024-02-04T14:08:56.775Z 6 120-139 182 182 182 311 311 311 F F F
197552024-02-04T14:08:56.775Z 7 140-159 183 183 183 314 314 314 F F F
197562024-02-04T14:08:56.775Z 8 160-179 183 183 183 314 314 314 F F F
197572024-02-04T14:08:56.775Z 9 180-199 182 182 182 311 311 311 F F F
197582024-02-04T14:08:56.775Z 10 200-219 177 177 177 303 303 303 F F F
197592024-02-04T14:08:56.775Z 11 220-239 182 182 182 313 313 313 F F F
197602024-02-04T14:08:56.775Z 12 240-259 182 182 182 313 313 313 F F F
197612024-02-04T14:08:56.775Z 13 260-279 182 182 182 312 312 312 F F F
197622024-02-04T14:08:56.775Z 14 280-299 182 182 182 312 312 312 F F F
197632024-02-04T14:08:56.775Z 15 300-319 181 181 181 310 310 310 F F F
197642024-02-04T14:08:56.775Z 16 320-339 182 182 182 311 311 311 F F F
197652024-02-04T14:08:56.775Z 17 340-359 176 176 176 302 302 302 F F F
197662024-02-04T14:08:56.775Z 18 360-379 178 178 178 306 306 306 F F F
197672024-02-04T14:08:56.781Z 19 380-399 179 179 179 307 307 307 F F F
197682024-02-04T14:08:56.781Z 20 400-419 179 179 179 307 307 307 F F F
197692024-02-04T14:08:56.781Z 21 420-439 182 182 182 311 311 311 F F F
197702024-02-04T14:08:56.781Z 22 440-459 181 181 181 310 310 310 F F F
197712024-02-04T14:08:56.781Z 23 460-479 183 183 183 314 314 314 F F F
197722024-02-04T14:08:56.781Z 24 480-499 182 182 182 311 311 311 F F F
197732024-02-04T14:08:56.781Z 25 500-519 181 181 181 310 310 310 F F F
197742024-02-04T14:08:56.781Z 26 520-539 181 181 181 310 310 310 F F F
197752024-02-04T14:08:56.781Z 27 540-559 182 182 182 311 311 311 F F F
197762024-02-04T14:08:56.781Z 28 560-579 178 178 178 305 305 305 F F F
197772024-02-04T14:08:56.781Z 29 580-599 182 182 182 311 311 311 F F F
197782024-02-04T14:08:56.781ZMax gen: 183, Max flush: 314
197792024-02-04T14:08:56.785Z
197802024-02-04T14:08:56.785ZBegin loop 92
197812024-02-04T14:08:56.785ZDownstairs to restart: 0
197822024-02-04T14:08:56.811Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.843155405Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868}
197832024-02-04T14:08:56.811Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.84361566Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868}
197842024-02-04T14:08:56.815Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.84368788Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868}
197852024-02-04T14:08:56.815Z{"msg":"Crucible d73308a2-be2e-4bb9-9e7e-9470e164e099 has session id: d8fb3e8d-da39-4c60-b94d-d27c29c0802e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.843731401Z","hostname":"Crucible runtime is spawned
197862024-02-04T14:08:56.815Zip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197872024-02-04T14:08:56.815Z{"msg":"Upstairs opts: Upstairs UUID: d73308a2-be2e-4bb9-9e7e-9470e164e099, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.843886013Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197882024-02-04T14:08:56.815Z{"msg":"Crucible stats registered with UUID: d73308a2-be2e-4bb9-9e7e-9470e164e099","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.843951656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197892024-02-04T14:08:56.815Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.843977358Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868}
197902024-02-04T14:08:56.815Z{"msg":"d73308a2-be2e-4bb9-9e7e-9470e164e099 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844109687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197912024-02-04T14:08:56.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844166619Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197922024-02-04T14:08:56.815Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844192196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197932024-02-04T14:08:56.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844214516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197942024-02-04T14:08:56.815Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844285779Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197952024-02-04T14:08:56.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844327156Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197962024-02-04T14:08:56.816Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844504486Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197972024-02-04T14:08:56.816Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844583196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197982024-02-04T14:08:56.816Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844631408Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
197992024-02-04T14:08:56.816Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844676773Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198002024-02-04T14:08:56.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844720231Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198012024-02-04T14:08:56.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844783692Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198022024-02-04T14:08:56.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.844810872Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198032024-02-04T14:08:56.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.84601087Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198042024-02-04T14:08:56.816Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.846083455Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198052024-02-04T14:08:56.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.846130675Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198062024-02-04T14:08:56.816Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.846269714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198072024-02-04T14:08:56.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.847152048Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198082024-02-04T14:08:56.821Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.847228227Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198092024-02-04T14:08:56.821Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.847375425Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198102024-02-04T14:08:56.821Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.847574962Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198112024-02-04T14:08:56.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.84824462Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198122024-02-04T14:08:56.821Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.848313218Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198132024-02-04T14:08:56.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.848340517Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198142024-02-04T14:08:56.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.848453668Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198152024-02-04T14:08:56.821Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.848506793Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198162024-02-04T14:08:56.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.848611628Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198172024-02-04T14:08:56.821Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.848723858Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198182024-02-04T14:08:56.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849365435Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198192024-02-04T14:08:56.821Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849432484Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198202024-02-04T14:08:56.821Z{"msg":"[0]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 311, 314, 314, 311, 303, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849461283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198212024-02-04T14:08:56.821Z{"msg":"[0]R generation[0..12]: [176, 179, 179, 178, 179, 179, 182, 183, 183, 182, 177, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849504791Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198222024-02-04T14:08:56.821Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849531373Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198232024-02-04T14:08:56.821Z{"msg":"[1]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 311, 314, 314, 311, 303, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849581134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198242024-02-04T14:08:56.821Z{"msg":"[1]R generation[0..12]: [176, 179, 179, 178, 179, 179, 182, 183, 183, 182, 177, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849606337Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198252024-02-04T14:08:56.821Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849656714Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198262024-02-04T14:08:56.821Z{"msg":"[2]R flush_numbers[0..12]: [301, 307, 307, 304, 307, 307, 311, 314, 314, 311, 303, 313]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
198272024-02-04T14:08:56.821Z,"time":"2024-02-04T14:08:55.849683064Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198282024-02-04T14:08:56.822Z{"msg":"[2]R generation[0..12]: [176, 179, 179, 178, 179, 179, 182, 183, 183, 182, 177, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849809862Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198292024-02-04T14:08:56.822Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849840707Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198302024-02-04T14:08:56.822Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849881857Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198312024-02-04T14:08:56.822Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849906269Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198322024-02-04T14:08:56.822Z{"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849946696Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198332024-02-04T14:08:56.822Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849970024Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198342024-02-04T14:08:56.822Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.849992989Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198352024-02-04T14:08:56.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.850013336Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198362024-02-04T14:08:56.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.850036053Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198372024-02-04T14:08:56.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.850058546Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198382024-02-04T14:08:56.822Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.850100717Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198392024-02-04T14:08:56.822Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.850123119Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198402024-02-04T14:08:56.822Z{"msg":"d73308a2-be2e-4bb9-9e7e-9470e164e099 is now active with session: d8fb3e8d-da39-4c60-b94d-d27c29c0802e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.850145378Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198412024-02-04T14:08:56.822Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.850184282Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868,"session_id":"d8fb3e8d-da39-4c60-b94d-d27c29c0802e"}
198422024-02-04T14:08:56.822Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:08:55.850235581Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868}
198432024-02-04T14:08:56.822ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
198442024-02-04T14:08:56.822ZLoading write count information from file "/tmp/test_repair_verify.data"
198452024-02-04T14:08:56.822ZRun Repair workload
198462024-02-04T14:08:56.822Z01/30 Write block 190 len 4096 data: 12
198472024-02-04T14:08:56.825Z02/30 Read block 371 len 8192
198482024-02-04T14:08:56.825Z03/30 Read block 421 len 36864
198492024-02-04T14:08:56.826Z04/30 Read block 82 len 8192
198502024-02-04T14:08:56.826Z05/30 Read block 559 len 32768
198512024-02-04T14:08:56.828Z06/30 Read block 343 len 36864
198522024-02-04T14:08:56.831Z07/30 Write block 82 len 24576 data: 20 17 17 15 16 12
198532024-02-04T14:08:56.834Z08/30 Write block 536 len 12288 data: 10 7 10
198542024-02-04T14:08:56.834Z09/30 Read block 393 len 4096
198552024-02-04T14:08:56.834Z10/30 Read block 160 len 8192
198562024-02-04T14:08:56.837Z11/30 Read block 1 len 20480
198572024-02-04T14:08:56.840Z12/30 Write block 461 len 40960 data: 18 15 16 15 18 21 18 14 15 13
198582024-02-04T14:08:56.840Z13/30 Write block 521 len 12288 data: 11 13 14
198592024-02-04T14:08:56.840Z14/30 Read block 228 len 4096
198602024-02-04T14:08:56.842Z15/30 Read block 94 len 28672
198612024-02-04T14:08:56.845Z16/30 Read block 500 len 36864
198622024-02-04T14:08:56.848Z17/30 Write block 164 len 36864 data: 14 16 17 18 14 13 14 12 16
198632024-02-04T14:08:56.848Z18/30 Read block 105 len 20480
198642024-02-04T14:08:56.851Z19/30 Write block 180 len 16384 data: 18 18 19 21
198652024-02-04T14:08:56.851Z20/30 Read block 258 len 40960
198662024-02-04T14:08:56.854Z21/30 Write block 152 len 36864 data: 13 12 11 9 8 9 11 13 12
198672024-02-04T14:08:56.854Z22/30 Read block 89 len 24576
198682024-02-04T14:08:56.857Z23/30 Read block 201 len 12288
198692024-02-04T14:08:56.857Z24/30 Read block 1 len 24576
198702024-02-04T14:08:56.857Z25/30 Read block 238 len 8192
198712024-02-04T14:08:56.860Z26/30 Flush
198722024-02-04T14:08:56.862Z27/30 Write block 339 len 4096 data: 12
198732024-02-04T14:08:56.862Z28/30 Read block 267 len 8192
198742024-02-04T14:08:56.866Z29/30 Write block 211 len 8192 data: 13 13
198752024-02-04T14:08:56.866Z30/30 Write block 282 len 32768 data: 9 13 18 16 13 15 17 17
198762024-02-04T14:08:56.866Z----------------------------------------------------------------
198772024-02-04T14:08:56.866Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
198782024-02-04T14:08:56.866ZStates: Active Active Active
198792024-02-04T14:08:56.866ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198802024-02-04T14:08:56.866Z 1 Acked 1000 Write 1 Done Done Done false
198812024-02-04T14:08:56.866Z 2 Acked 1001 Read 2 Done Done Done false
198822024-02-04T14:08:56.866Z 3 Acked 1002 Read 9 Done Done Done false
198832024-02-04T14:08:56.866Z 4 Acked 1003 Read 2 Done Done Done false
198842024-02-04T14:08:56.866Z 5 Acked 1004 Read 8 Done Done Done false
198852024-02-04T14:08:56.866Z 6 Acked 1005 Read 9 Done Done Done false
198862024-02-04T14:08:56.866Z 7 Acked 1006 Write 6 Sent Done Done false
198872024-02-04T14:08:56.866Z 8 Acked 1007 Write 3 Done Done Done false
198882024-02-04T14:08:56.866Z 9 Acked 1008 Read 1 Done Done Done false
198892024-02-04T14:08:56.866Z 10 Acked 1009 Read 2 Done Done Done false
198902024-02-04T14:08:56.867Z 11 Acked 1010 Read 5 Done Done Done false
198912024-02-04T14:08:56.867Z 12 Acked 1011 Write 10 Sent Done Done false
198922024-02-04T14:08:56.867Z 13 Acked 1012 Write 3 Sent Done Done false
198932024-02-04T14:08:56.867Z 14 Acked 1013 Read 1 Sent Done Done false
198942024-02-04T14:08:56.867Z 15 Acked 1014 Read 7 Sent Done Done false
198952024-02-04T14:08:56.867Z 16 Acked 1015 Read 9 Sent Done Done false
198962024-02-04T14:08:56.867Z 17 Acked 1016 Write 9 Sent Done Done false
198972024-02-04T14:08:56.867Z 18 Acked 1017 Read 5 Sent Done Done false
198982024-02-04T14:08:56.867Z 19 Acked 1018 Write 4 Sent Done Done false
198992024-02-04T14:08:56.867Z 20 Acked 1019 Read 10 Sent Done Done false
199002024-02-04T14:08:56.867Z 21 Acked 1020 Write 9 Sent Done Done false
199012024-02-04T14:08:56.867Z 22 Acked 1021 Read 6 Sent Done Done false
199022024-02-04T14:08:56.867Z 23 Acked 1022 Read 3 Sent Done Done false
199032024-02-04T14:08:56.867Z 24 Acked 1023 Read 6 Sent Done Done false
199042024-02-04T14:08:56.867Z 25 Acked 1024 Read 2 Sent Done Done false
199052024-02-04T14:08:56.867Z 26 Acked 1025 Flush 0 Sent Done Done false
199062024-02-04T14:08:56.867Z 27 Acked 1026 Write 1 Sent Done Done false
199072024-02-04T14:08:56.867Z 28 Acked 1027 Read 2 Sent Done Done false
199082024-02-04T14:08:56.867Z 29 Acked 1028 Write 2 Sent Done Done false
199092024-02-04T14:08:56.867Z 30 Acked 1029 Write 8 Sent Sent Sent false
199102024-02-04T14:08:56.867Z STATES DS:0 DS:1 DS:2 TOTAL
199112024-02-04T14:08:56.867Z New 0 0 0 0
199122024-02-04T14:08:56.867Z Sent 20 1 1 22
199132024-02-04T14:08:56.867Z Done 10 29 29 68
199142024-02-04T14:08:56.868Z Skipped 0 0 0 0
199152024-02-04T14:08:56.868Z Error 0 0 0 0
199162024-02-04T14:08:56.868ZLast Flush: 0 1025 1025
199172024-02-04T14:08:56.868ZDownstairs last five completed:
199182024-02-04T14:08:56.868ZUpstairs last five completed: 30 29 28 27 26
199192024-02-04T14:08:56.871Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:08:55.898518836Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1868}
199202024-02-04T14:08:56.871ZWrote out file "/tmp/test_repair_verify.data"
199212024-02-04T14:08:56.874Z
199222024-02-04T14:08:56.878ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
199232024-02-04T14:08:56.906ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199242024-02-04T14:08:56.906Z 0 000-019 176 176 176 301 301 301 F F F
199252024-02-04T14:08:56.906Z 1 020-039 179 179 179 307 307 307 F F F
199262024-02-04T14:08:56.906Z 2 040-059 179 179 179 307 307 307 F F F
199272024-02-04T14:08:56.906Z 3 060-079 178 178 178 304 304 304 F F F
199282024-02-04T14:08:56.906Z 4 080-099 179 184 184 307 315 315 F F F <---
199292024-02-04T14:08:56.909Z 5 100-119 179 179 179 307 307 307 F F F
199302024-02-04T14:08:56.909Z 6 120-139 182 182 182 311 311 311 F F F
199312024-02-04T14:08:56.909Z 7 140-159 183 184 184 314 315 315 F F F <---
199322024-02-04T14:08:56.909Z 8 160-179 183 184 184 314 315 315 F F F <---
199332024-02-04T14:08:56.909Z 9 180-199 182 184 184 311 315 315 T F F <---
199342024-02-04T14:08:56.909Z 10 200-219 177 177 177 303 303 303 F T T <---
199352024-02-04T14:08:56.909Z 11 220-239 182 182 182 313 313 313 F F F
199362024-02-04T14:08:56.909Z 12 240-259 182 182 182 313 313 313 F F F
199372024-02-04T14:08:56.909Z 13 260-279 182 182 182 312 312 312 F F F
199382024-02-04T14:08:56.909Z 14 280-299 182 182 182 312 312 312 F T T <---
199392024-02-04T14:08:56.909Z 15 300-319 181 181 181 310 310 310 F F F
199402024-02-04T14:08:56.910Z 16 320-339 182 182 182 311 311 311 F T T <---
199412024-02-04T14:08:56.910Z 17 340-359 176 176 176 302 302 302 F F F
199422024-02-04T14:08:56.910Z 18 360-379 178 178 178 306 306 306 F F F
199432024-02-04T14:08:56.910Z 19 380-399 179 179 179 307 307 307 F F F
199442024-02-04T14:08:56.910Z 20 400-419 179 179 179 307 307 307 F F F
199452024-02-04T14:08:56.910Z 21 420-439 182 182 182 311 311 311 F F F
199462024-02-04T14:08:56.910Z 22 440-459 181 181 181 310 310 310 F F F
199472024-02-04T14:08:56.910Z 23 460-479 183 184 184 314 315 315 F F F <---
199482024-02-04T14:08:56.910Z 24 480-499 182 182 182 311 311 311 F F F
199492024-02-04T14:08:56.910Z 25 500-519 181 181 181 310 310 310 F F F
199502024-02-04T14:08:56.910Z 26 520-539 181 184 184 310 315 315 T F F <---
199512024-02-04T14:08:56.910Z 27 540-559 182 182 182 311 311 311 F F F
199522024-02-04T14:08:56.910Z 28 560-579 178 178 178 305 305 305 F F F
199532024-02-04T14:08:56.910Z 29 580-599 182 182 182 311 311 311 F F F
199542024-02-04T14:08:56.910ZMax gen: 184, Max flush: 315
199552024-02-04T14:08:56.910ZError: Difference in extent metadata found!
199562024-02-04T14:08:56.910ZA Difference in extent metadata is expected here
199572024-02-04T14:08:56.910ZOn loop 92
199582024-02-04T14:08:56.910Z
199592024-02-04T14:08:56.913ZVerifying data now
199602024-02-04T14:09:02.746ZNo change after verify
199612024-02-04T14:09:02.746ZLoop: 92 Downstairs dump after verify (and repair):
199622024-02-04T14:09:02.772ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199632024-02-04T14:09:02.772Z 0 000-019 176 176 176 301 301 301 F F F
199642024-02-04T14:09:02.772Z 1 020-039 179 179 179 307 307 307 F F F
199652024-02-04T14:09:02.772Z 2 040-059 179 179 179 307 307 307 F F F
199662024-02-04T14:09:02.772Z 3 060-079 178 178 178 304 304 304 F F F
199672024-02-04T14:09:02.772Z 4 080-099 184 184 184 315 315 315 F F F
199682024-02-04T14:09:02.772Z 5 100-119 179 179 179 307 307 307 F F F
199692024-02-04T14:09:02.772Z 6 120-139 182 182 182 311 311 311 F F F
199702024-02-04T14:09:02.772Z 7 140-159 184 184 184 315 315 315 F F F
199712024-02-04T14:09:02.772Z 8 160-179 184 184 184 315 315 315 F F F
199722024-02-04T14:09:02.772Z 9 180-199 184 184 184 315 315 315 F F F
199732024-02-04T14:09:02.772Z 10 200-219 185 185 185 316 316 316 F F F
199742024-02-04T14:09:02.772Z 11 220-239 182 182 182 313 313 313 F F F
199752024-02-04T14:09:02.772Z 12 240-259 182 182 182 313 313 313 F F F
199762024-02-04T14:09:02.772Z 13 260-279 182 182 182 312 312 312 F F F
199772024-02-04T14:09:02.772Z 14 280-299 185 185 185 316 316 316 F F F
199782024-02-04T14:09:02.776Z 15 300-319 181 181 181 310 310 310 F F F
199792024-02-04T14:09:02.776Z 16 320-339 185 185 185 316 316 316 F F F
199802024-02-04T14:09:02.776Z 17 340-359 176 176 176 302 302 302 F F F
199812024-02-04T14:09:02.776Z 18 360-379 178 178 178 306 306 306 F F F
199822024-02-04T14:09:02.776Z 19 380-399 179 179 179 307 307 307 F F F
199832024-02-04T14:09:02.776Z 20 400-419 179 179 179 307 307 307 F F F
199842024-02-04T14:09:02.776Z 21 420-439 182 182 182 311 311 311 F F F
199852024-02-04T14:09:02.776Z 22 440-459 181 181 181 310 310 310 F F F
199862024-02-04T14:09:02.777Z 23 460-479 184 184 184 315 315 315 F F F
199872024-02-04T14:09:02.777Z 24 480-499 182 182 182 311 311 311 F F F
199882024-02-04T14:09:02.777Z 25 500-519 181 181 181 310 310 310 F F F
199892024-02-04T14:09:02.777Z 26 520-539 184 184 184 315 315 315 F F F
199902024-02-04T14:09:02.777Z 27 540-559 182 182 182 311 311 311 F F F
199912024-02-04T14:09:02.777Z 28 560-579 178 178 178 305 305 305 F F F
199922024-02-04T14:09:02.777Z 29 580-599 182 182 182 311 311 311 F F F
199932024-02-04T14:09:02.777ZMax gen: 185, Max flush: 316
199942024-02-04T14:09:02.777Z
199952024-02-04T14:09:02.777ZBegin loop 93
199962024-02-04T14:09:02.777ZDownstairs to restart: 1
199972024-02-04T14:09:02.805Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.836295165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876}
199982024-02-04T14:09:02.810Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.836913773Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876}
199992024-02-04T14:09:02.810Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.836993988Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876}
200002024-02-04T14:09:02.810Z{Crucible runtime is spawned
200012024-02-04T14:09:02.810Z"msg":"Crucible d19d9f48-9bb4-4b33-adcd-503f8d03047e has session id: ea4e0ab8-085b-48eb-8761-b5512955b72c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.83712043Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200022024-02-04T14:09:02.810Z{"msg":"Upstairs opts: Upstairs UUID: d19d9f48-9bb4-4b33-adcd-503f8d03047e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837231708Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200032024-02-04T14:09:02.810Z{"msg":"Crucible stats registered with UUID: d19d9f48-9bb4-4b33-adcd-503f8d03047e","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837293368Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200042024-02-04T14:09:02.810Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837318885Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876}
200052024-02-04T14:09:02.810Z{"msg":"d19d9f48-9bb4-4b33-adcd-503f8d03047e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837345939Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200062024-02-04T14:09:02.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837367756Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200072024-02-04T14:09:02.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837419945Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200082024-02-04T14:09:02.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837445038Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200092024-02-04T14:09:02.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837473769Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200102024-02-04T14:09:02.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837516573Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200112024-02-04T14:09:02.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837561215Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200122024-02-04T14:09:02.811Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837635785Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200132024-02-04T14:09:02.811Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837710924Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200142024-02-04T14:09:02.811Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.837783171Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200152024-02-04T14:09:02.811Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.838006006Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200162024-02-04T14:09:02.811Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.838058481Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200172024-02-04T14:09:02.811Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.83815787Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200182024-02-04T14:09:02.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.839439514Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200192024-02-04T14:09:02.811Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.839509837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200202024-02-04T14:09:02.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.839536331Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200212024-02-04T14:09:02.811Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.839604594Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200222024-02-04T14:09:02.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.840566954Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200232024-02-04T14:09:02.811Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.840837949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200242024-02-04T14:09:02.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.840952443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200252024-02-04T14:09:02.811Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.840995605Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200262024-02-04T14:09:02.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.841464639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200272024-02-04T14:09:02.819Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.84157746Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200282024-02-04T14:09:02.819Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.841655379Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200292024-02-04T14:09:02.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.842073359Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200302024-02-04T14:09:02.819Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.842144189Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200312024-02-04T14:09:02.819Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.842170169Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200322024-02-04T14:09:02.819Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.842192544Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200332024-02-04T14:09:02.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.842888902Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200342024-02-04T14:09:02.819Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.842969912Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200352024-02-04T14:09:02.819Z{"msg":"[0]R flush_numbers[0..12]: [301, 307, 307, 304, 315, 307, 311, 315, 315, 315, 316, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.8430261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200362024-02-04T14:09:02.819Z{"msg":"[0]R generation[0..12]: [176, 179, 179, 178, 184, 179, 182, 184, 184, 184, 185, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843060552Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200372024-02-04T14:09:02.819Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843086543Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200382024-02-04T14:09:02.819Z{"msg":"[1]R flush_numbers[0..12]: [301, 307, 307, 304, 315, 307, 311, 315, 315, 315, 316, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843131111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200392024-02-04T14:09:02.819Z{"msg":"[1]R generation[0..12]: [176, 179, 179, 178, 184, 179, 182, 184, 184, 184, 185, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843174318Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200402024-02-04T14:09:02.820Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843199438Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876Wait for a query_work_queue command to finish before sending IO
200412024-02-04T14:09:02.820Z,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200422024-02-04T14:09:02.820Z{"msg":"[2]R flush_numbers[0..12]: [301, 307, 307, 304, 315, 307, 311, 315, 315, 315, 316, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.84322672Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200432024-02-04T14:09:02.820Z{"msg":"[2]R generation[0..12]: [176, 179, 179, 178, 184, 179, 182, 184, 184, 184, 185, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843258591Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200442024-02-04T14:09:02.820Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843283852Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200452024-02-04T14:09:02.820Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843306041Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200462024-02-04T14:09:02.820Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843354674Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200472024-02-04T14:09:02.820Z{"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843377582Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200482024-02-04T14:09:02.820Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843399345Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200492024-02-04T14:09:02.820Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843422017Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200502024-02-04T14:09:02.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843460837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200512024-02-04T14:09:02.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843484056Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200522024-02-04T14:09:02.820Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843531559Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200532024-02-04T14:09:02.820Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843555106Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200542024-02-04T14:09:02.820Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843605503Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200552024-02-04T14:09:02.820Z{"msg":"d19d9f48-9bb4-4b33-adcd-503f8d03047e is now active with session: ea4e0ab8-085b-48eb-8761-b5512955b72c","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843628759Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200562024-02-04T14:09:02.820Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843650464Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876,"session_id":"ea4e0ab8-085b-48eb-8761-b5512955b72c"}
200572024-02-04T14:09:02.820Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:01.843694907Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876}
200582024-02-04T14:09:02.820ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200592024-02-04T14:09:02.820ZLoading write count information from file "/tmp/test_repair_verify.data"
200602024-02-04T14:09:02.820ZRun Repair workload
200612024-02-04T14:09:02.820Z01/30 Write block 444 len 28672 data: 18 19 21 20 19 17 15
200622024-02-04T14:09:02.820Z02/30 Write block 470 len 8192 data: 14 12
200632024-02-04T14:09:02.820Z03/30 Read block 38 len 20480
200642024-02-04T14:09:02.824Z04/30 Flush
200652024-02-04T14:09:02.824Z05/30 Write block 459 len 40960 data: 20 19 19 16 17 16 19 22 19 15
200662024-02-04T14:09:02.824Z06/30 Write block 30 len 40960 data: 13 13 11 15 14 18 20 18 15 14
200672024-02-04T14:09:02.824Z07/30 Read block 536 len 32768
200682024-02-04T14:09:02.827Z08/30 Read block 55 len 12288
200692024-02-04T14:09:02.830Z09/30 Write block 422 len 16384 data: 12 12 7 9
200702024-02-04T14:09:02.830Z10/30 Flush
200712024-02-04T14:09:02.836Z11/30 Write block 188 len 4096 data: 15
200722024-02-04T14:09:02.836Z12/30 Read block 491 len 4096
200732024-02-04T14:09:02.836Z13/30 Read block 354 len 28672
200742024-02-04T14:09:02.839Z14/30 Write block 497 len 28672 data: 14 16 18 18 17 13 13
200752024-02-04T14:09:02.839Z15/30 Read block 338 len 40960
200762024-02-04T14:09:02.839Z16/30 Write block 309 len 36864 data: 14 16 15 11 12 12 8 8 9
200772024-02-04T14:09:02.843Z17/30 Read block 547 len 8192
200782024-02-04T14:09:02.843Z18/30 Write block 305 len 4096 data: 17
200792024-02-04T14:09:02.843Z19/30 Read block 100 len 16384
200802024-02-04T14:09:02.847Z20/30 Read block 477 len 4096
200812024-02-04T14:09:02.847Z21/30 Read block 589 len 8192
200822024-02-04T14:09:02.847Z22/30 Read block 276 len 36864
200832024-02-04T14:09:02.851Z23/30 Read block 566 len 20480
200842024-02-04T14:09:02.851Z24/30 Write block 289 len 8192 data: 18 17
200852024-02-04T14:09:02.851Z25/30 Read block 488 len 28672
200862024-02-04T14:09:02.851Z26/30 Read block 34 len 4096
200872024-02-04T14:09:02.854Z27/30 Read block 151 len 8192
200882024-02-04T14:09:02.855Z28/30 Write block 206 len 32768 data: 17 16 14 13 11 14 14 10
200892024-02-04T14:09:02.855Z29/30 Read block 444 len 16384
200902024-02-04T14:09:02.861Z30/30 Write block 340 len 32768 data: 14 14 12 12 12 14 13 14
200912024-02-04T14:09:02.861Z----------------------------------------------------------------
200922024-02-04T14:09:02.861Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:26
200932024-02-04T14:09:02.861ZStates: Active Active Active
200942024-02-04T14:09:02.861ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200952024-02-04T14:09:02.861Z 5 Acked 1004 Write 10 Done Done Done false
200962024-02-04T14:09:02.861Z 6 Acked 1005 Write 10 Done Sent Done false
200972024-02-04T14:09:02.861Z 7 Acked 1006 Read 8 Done Sent Done false
200982024-02-04T14:09:02.861Z 8 Acked 1007 Read 3 Done Sent Done false
200992024-02-04T14:09:02.861Z 9 Acked 1008 Write 4 Done Sent Done false
201002024-02-04T14:09:02.862Z 10 Acked 1009 Flush 0 Done Sent Done false
201012024-02-04T14:09:02.862Z 11 Acked 1010 Write 1 Done Sent Done false
201022024-02-04T14:09:02.862Z 12 Acked 1011 Read 1 Done Sent Done false
201032024-02-04T14:09:02.862Z 13 Acked 1012 Read 7 Done Sent Done false
201042024-02-04T14:09:02.862Z 14 Acked 1013 Write 7 Done Sent Done false
201052024-02-04T14:09:02.862Z 15 Acked 1014 Read 10 Done Sent Done false
201062024-02-04T14:09:02.862Z 16 Acked 1015 Write 9 Done Sent Done false
201072024-02-04T14:09:02.862Z 17 Acked 1016 Read 2 Done Sent Done false
201082024-02-04T14:09:02.862Z 18 Acked 1017 Write 1 Done Sent Done false
201092024-02-04T14:09:02.862Z 19 Acked 1018 Read 4 Done Sent Done false
201102024-02-04T14:09:02.862Z 20 Acked 1019 Read 1 Done Sent Done false
201112024-02-04T14:09:02.862Z 21 Acked 1020 Read 2 Done Sent Done false
201122024-02-04T14:09:02.862Z 22 Acked 1021 Read 9 Done Sent Done false
201132024-02-04T14:09:02.862Z 23 Acked 1022 Read 5 Done Sent Done false
201142024-02-04T14:09:02.862Z 24 Acked 1023 Write 2 Done Sent Done false
201152024-02-04T14:09:02.862Z 25 Acked 1024 Read 7 Done Sent Done false
201162024-02-04T14:09:02.862Z 26 Acked 1025 Read 1 Done Sent Done false
201172024-02-04T14:09:02.862Z 27 Acked 1026 Read 2 Done Sent Done false
201182024-02-04T14:09:02.862Z 28 Acked 1027 Write 8 Done Sent Done false
201192024-02-04T14:09:02.862Z 29 Acked 1028 Read 4 Done Sent Done false
201202024-02-04T14:09:02.862Z 30 Acked 1029 Write 8 Sent Sent Sent false
201212024-02-04T14:09:02.862Z STATES DS:0 DS:1 DS:2 TOTAL
201222024-02-04T14:09:02.862Z New 0 0 0 0
201232024-02-04T14:09:02.862Z Sent 1 25 1 27
201242024-02-04T14:09:02.862Z Done 25 1 25 51
201252024-02-04T14:09:02.863Z Skipped 0 0 0 0
201262024-02-04T14:09:02.863Z Error 0 0 0 0
201272024-02-04T14:09:02.863ZLast Flush: 1009 1003 1009
201282024-02-04T14:09:02.863ZDownstairs last five completed: 1003 1002 1001 1000
201292024-02-04T14:09:02.863ZUpstairs last five completed: 30 29 28 27 26
201302024-02-04T14:09:02.863Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:09:01.889913523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1876}
201312024-02-04T14:09:02.863ZWrote out file "/tmp/test_repair_verify.data"
201322024-02-04T14:09:02.868Z
201332024-02-04T14:09:02.871ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
201342024-02-04T14:09:02.899ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201352024-02-04T14:09:02.899Z 0 000-019 176 176 176 301 301 301 F F F
201362024-02-04T14:09:02.899Z 1 020-039 186 179 186 318 307 318 F F F <---
201372024-02-04T14:09:02.899Z 2 040-059 179 179 179 307 307 307 F F F
201382024-02-04T14:09:02.903Z 3 060-079 178 178 178 304 304 304 F F F
201392024-02-04T14:09:02.903Z 4 080-099 184 184 184 315 315 315 F F F
201402024-02-04T14:09:02.903Z 5 100-119 179 179 179 307 307 307 F F F
201412024-02-04T14:09:02.903Z 6 120-139 182 182 182 311 311 311 F F F
201422024-02-04T14:09:02.903Z 7 140-159 184 184 184 315 315 315 F F F
201432024-02-04T14:09:02.903Z 8 160-179 184 184 184 315 315 315 F F F
201442024-02-04T14:09:02.903Z 9 180-199 184 184 184 315 315 315 T F T <---
201452024-02-04T14:09:02.903Z 10 200-219 185 185 185 316 316 316 T F T <---
201462024-02-04T14:09:02.903Z 11 220-239 182 182 182 313 313 313 F F F
201472024-02-04T14:09:02.903Z 12 240-259 182 182 182 313 313 313 F F F
201482024-02-04T14:09:02.903Z 13 260-279 182 182 182 312 312 312 F F F
201492024-02-04T14:09:02.904Z 14 280-299 185 185 185 316 316 316 T F T <---
201502024-02-04T14:09:02.904Z 15 300-319 181 181 181 310 310 310 T F T <---
201512024-02-04T14:09:02.904Z 16 320-339 185 185 185 316 316 316 F F F
201522024-02-04T14:09:02.904Z 17 340-359 176 176 176 302 302 302 T F T <---
201532024-02-04T14:09:02.904Z 18 360-379 178 178 178 306 306 306 F F F
201542024-02-04T14:09:02.904Z 19 380-399 179 179 179 307 307 307 F F F
201552024-02-04T14:09:02.904Z 20 400-419 179 179 179 307 307 307 F F F
201562024-02-04T14:09:02.904Z 21 420-439 186 182 186 318 311 318 F F F <---
201572024-02-04T14:09:02.904Z 22 440-459 186 186 186 318 317 318 F T F <---
201582024-02-04T14:09:02.904Z 23 460-479 186 186 186 318 317 318 F T F <---
201592024-02-04T14:09:02.904Z 24 480-499 182 182 182 311 311 311 T F T <---
201602024-02-04T14:09:02.904Z 25 500-519 181 181 181 310 310 310 T F T <---
201612024-02-04T14:09:02.904Z 26 520-539 184 184 184 315 315 315 F F F
201622024-02-04T14:09:02.904Z 27 540-559 182 182 182 311 311 311 F F F
201632024-02-04T14:09:02.904Z 28 560-579 178 178 178 305 305 305 F F F
201642024-02-04T14:09:02.904Z 29 580-599 182 182 182 311 311 311 F F F
201652024-02-04T14:09:02.904ZMax gen: 186, Max flush: 318
201662024-02-04T14:09:02.904ZError: Difference in extent metadata found!
201672024-02-04T14:09:02.909ZA Difference in extent metadata is expected here
201682024-02-04T14:09:02.909ZOn loop 93
201692024-02-04T14:09:02.909Z
201702024-02-04T14:09:02.909ZVerifying data now
201712024-02-04T14:09:07.904ZNo change after verify
201722024-02-04T14:09:07.904ZLoop: 93 Downstairs dump after verify (and repair):
201732024-02-04T14:09:07.927ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201742024-02-04T14:09:07.927Z 0 000-019 176 176 176 301 301 301 F F F
201752024-02-04T14:09:07.927Z 1 020-039 186 186 186 318 318 318 F F F
201762024-02-04T14:09:07.927Z 2 040-059 179 179 179 307 307 307 F F F
201772024-02-04T14:09:07.927Z 3 060-079 178 178 178 304 304 304 F F F
201782024-02-04T14:09:07.927Z 4 080-099 184 184 184 315 315 315 F F F
201792024-02-04T14:09:07.927Z 5 100-119 179 179 179 307 307 307 F F F
201802024-02-04T14:09:07.931Z 6 120-139 182 182 182 311 311 311 F F F
201812024-02-04T14:09:07.931Z 7 140-159 184 184 184 315 315 315 F F F
201822024-02-04T14:09:07.932Z 8 160-179 184 184 184 315 315 315 F F F
201832024-02-04T14:09:07.932Z 9 180-199 187 187 187 319 319 319 F F F
201842024-02-04T14:09:07.932Z 10 200-219 187 187 187 319 319 319 F F F
201852024-02-04T14:09:07.932Z 11 220-239 182 182 182 313 313 313 F F F
201862024-02-04T14:09:07.932Z 12 240-259 182 182 182 313 313 313 F F F
201872024-02-04T14:09:07.932Z 13 260-279 182 182 182 312 312 312 F F F
201882024-02-04T14:09:07.932Z 14 280-299 187 187 187 319 319 319 F F F
201892024-02-04T14:09:07.932Z 15 300-319 187 187 187 319 319 319 F F F
201902024-02-04T14:09:07.932Z 16 320-339 185 185 185 316 316 316 F F F
201912024-02-04T14:09:07.932Z 17 340-359 187 187 187 319 319 319 F F F
201922024-02-04T14:09:07.932Z 18 360-379 178 178 178 306 306 306 F F F
201932024-02-04T14:09:07.932Z 19 380-399 179 179 179 307 307 307 F F F
201942024-02-04T14:09:07.933Z 20 400-419 179 179 179 307 307 307 F F F
201952024-02-04T14:09:07.933Z 21 420-439 186 186 186 318 318 318 F F F
201962024-02-04T14:09:07.933Z 22 440-459 186 186 186 318 318 318 F F F
201972024-02-04T14:09:07.933Z 23 460-479 186 186 186 318 318 318 F F F
201982024-02-04T14:09:07.933Z 24 480-499 187 187 187 319 319 319 F F F
201992024-02-04T14:09:07.933Z 25 500-519 187 187 187 319 319 319 F F F
202002024-02-04T14:09:07.933Z 26 520-539 184 184 184 315 315 315 F F F
202012024-02-04T14:09:07.933Z 27 540-559 182 182 182 311 311 311 F F F
202022024-02-04T14:09:07.933Z 28 560-579 178 178 178 305 305 305 F F F
202032024-02-04T14:09:07.933Z 29 580-599 182 182 182 311 311 311 F F F
202042024-02-04T14:09:07.933ZMax gen: 187, Max flush: 319
202052024-02-04T14:09:07.937Z
202062024-02-04T14:09:07.937ZBegin loop 94
202072024-02-04T14:09:07.937ZDownstairs to restart: 0
202082024-02-04T14:09:07.961Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.9935189Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884}
202092024-02-04T14:09:07.967Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994051908Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884}
202102024-02-04T14:09:07.967Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994152233Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884}
202112024-02-04T14:09:07.967Z{Crucible runtime is spawned
202122024-02-04T14:09:07.967Z"msg":"Crucible 7826a0f0-8812-4b72-a876-16a03012c6e5 has session id: 3bcfad70-6f28-47d2-bc66-960f984f5fa8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994290686Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202132024-02-04T14:09:07.967Z{"msg":"Upstairs opts: Upstairs UUID: 7826a0f0-8812-4b72-a876-16a03012c6e5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994351134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202142024-02-04T14:09:07.967Z{"msg":"Crucible stats registered with UUID: 7826a0f0-8812-4b72-a876-16a03012c6e5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994413442Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202152024-02-04T14:09:07.967Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994453227Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884}
202162024-02-04T14:09:07.967Z{"msg":"7826a0f0-8812-4b72-a876-16a03012c6e5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994503333Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202172024-02-04T14:09:07.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.99452733Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202182024-02-04T14:09:07.967Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.99460586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202192024-02-04T14:09:07.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994650411Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202202024-02-04T14:09:07.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994743344Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202212024-02-04T14:09:07.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994821913Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202222024-02-04T14:09:07.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.99490713Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202232024-02-04T14:09:07.968Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.994989938Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202242024-02-04T14:09:07.968Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.995067656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202252024-02-04T14:09:07.968Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.99511524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202262024-02-04T14:09:07.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.995178466Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202272024-02-04T14:09:07.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.995205307Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202282024-02-04T14:09:07.968Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.995328977Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202292024-02-04T14:09:07.968Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.99636887Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202302024-02-04T14:09:07.968Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.996444903Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202312024-02-04T14:09:07.968Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.996487489Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202322024-02-04T14:09:07.968Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.996532283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202332024-02-04T14:09:07.969Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.99738012Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202342024-02-04T14:09:07.974Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.997596191Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202352024-02-04T14:09:07.974Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.997767654Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202362024-02-04T14:09:07.974Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.997814134Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202372024-02-04T14:09:07.974Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.998144659Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202382024-02-04T14:09:07.974Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.998205048Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202392024-02-04T14:09:07.974Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.99823094Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202402024-02-04T14:09:07.974Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.998621434Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202412024-02-04T14:09:07.974Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.998693848Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202422024-02-04T14:09:07.975Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.998719179Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202432024-02-04T14:09:07.975Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.99923003Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202442024-02-04T14:09:07.975Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:06.999973515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202452024-02-04T14:09:07.975Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000042522Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202462024-02-04T14:09:07.975Z{"msg":"[0]R flush_numbers[0..12]: [301, 318, 307, 304, 315, 307, 311, 315, 315, 319, 319, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000071495Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202472024-02-04T14:09:07.975Z{"msg":"[0]R generation[0..12]: [176, 186, 179, 178, 184, 179, 182, 184, 184, 187, 187, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000115297Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202482024-02-04T14:09:07.975Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000141663Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202492024-02-04T14:09:07.975Z{"msg":"[1]R flush_numbers[0..12]: [301, 318, 307, 304, 315, 307, 311, 315, 315, 319, 319, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000179534Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202502024-02-04T14:09:07.975Z{"msg":"[1]R generation[0..12]: [176, 186, 179, 178, 184, 179, 182, 184, 184, 187, 187, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000203917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202512024-02-04T14:09:07.975Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.00022846Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202522024-02-04T14:09:07.975Z{"msg":"[2]R flush_numbers[0..12]: [301, 318, 307, 304, 315, 307, 311, 315, 315, 319, 319, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000253198Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202532024-02-04T14:09:07.975Z{"msg":"[2]R generation[0..12]: [176, 186, 179, 178, 184, 179, 182, 184, 184, 187, 187, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000276989Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202542024-02-04T14:09:07.975Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000301281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202552024-02-04T14:09:07.976Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000323577Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202562024-02-04T14:09:07.976Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000346296Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202572024-02-04T14:09:07.976Z{"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000368257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202582024-02-04T14:09:07.976Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000390568Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202592024-02-04T14:09:07.976Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000413407Z","hostnameWait for a query_work_queue command to finish before sending IO
202602024-02-04T14:09:07.976Z":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202612024-02-04T14:09:07.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000440599Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202622024-02-04T14:09:07.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000496853Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202632024-02-04T14:09:07.976Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000530085Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202642024-02-04T14:09:07.976Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000575608Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202652024-02-04T14:09:07.976Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000598582Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202662024-02-04T14:09:07.976Z{"msg":"7826a0f0-8812-4b72-a876-16a03012c6e5 is now active with session: 3bcfad70-6f28-47d2-bc66-960f984f5fa8","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000620899Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202672024-02-04T14:09:07.976Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000659467Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884,"session_id":"3bcfad70-6f28-47d2-bc66-960f984f5fa8"}
202682024-02-04T14:09:07.977Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:07.000681781Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884}
202692024-02-04T14:09:07.977ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202702024-02-04T14:09:07.977ZLoading write count information from file "/tmp/test_repair_verify.data"
202712024-02-04T14:09:07.977ZRun Repair workload
202722024-02-04T14:09:07.977Z01/30 Write block 53 len 32768 data: 13 11 11 12 13 11 13 15
202732024-02-04T14:09:07.992Z02/30 Write block 427 len 20480 data: 9 9 9 12 13
202742024-02-04T14:09:07.992Z03/30 Write block 299 len 24576 data: 12 10 8 10 9 12
202752024-02-04T14:09:07.992Z04/30 Read block 363 len 40960
202762024-02-04T14:09:07.992Z05/30 Write block 270 len 4096 data: 19
202772024-02-04T14:09:07.992Z06/30 Write block 140 len 4096 data: 18
202782024-02-04T14:09:07.992Z07/30 Read block 20 len 32768
202792024-02-04T14:09:07.997Z08/30 Write block 111 len 40960 data: 15 19 16 18 18 15 15 12 13 13
202802024-02-04T14:09:07.997Z09/30 Write block 295 len 32768 data: 10 8 10 12 13 11 9 11
202812024-02-04T14:09:07.997Z10/30 Write block 293 len 8192 data: 11 8
202822024-02-04T14:09:07.997Z11/30 Read block 542 len 8192
202832024-02-04T14:09:07.997Z12/30 Write block 238 len 32768 data: 17 17 19 17 14 14 15 15
202842024-02-04T14:09:07.997Z13/30 Read block 119 len 40960
202852024-02-04T14:09:07.998Z14/30 Write block 510 len 40960 data: 16 13 8 6 8 13 13 13 13 13
202862024-02-04T14:09:07.998Z15/30 Read block 292 len 16384
202872024-02-04T14:09:07.998Z16/30 Write block 427 len 36864 data: 10 10 10 13 14 13 13 17 18
202882024-02-04T14:09:07.998Z17/30 Flush
202892024-02-04T14:09:08.002Z18/30 Write block 428 len 28672 data: 11 11 14 15 14 14 18
202902024-02-04T14:09:08.006Z19/30 Read block 343 len 36864
202912024-02-04T14:09:08.006Z20/30 Read block 596 len 8192
202922024-02-04T14:09:08.006Z21/30 Read block 355 len 16384
202932024-02-04T14:09:08.012Z22/30 Read block 547 len 24576
202942024-02-04T14:09:08.012Z23/30 Read block 451 len 24576
202952024-02-04T14:09:08.012Z24/30 Read block 489 len 40960
202962024-02-04T14:09:08.012Z25/30 Write block 590 len 20480 data: 10 8 9 10 12
202972024-02-04T14:09:08.016Z26/30 Read block 461 len 32768
202982024-02-04T14:09:08.016Z27/30 Read block 368 len 24576
202992024-02-04T14:09:08.020Z28/30 Write block 107 len 28672 data: 9 14 13 14 16 20 17
203002024-02-04T14:09:08.020Z29/30 Write block 394 len 20480 data: 14 14 12 15 13
203012024-02-04T14:09:08.020Z30/30 Write block 439 len 16384 data: 10 14 13 14
203022024-02-04T14:09:08.020Z----------------------------------------------------------------
203032024-02-04T14:09:08.020Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
203042024-02-04T14:09:08.020ZStates: Active Active Active
203052024-02-04T14:09:08.020ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
203062024-02-04T14:09:08.020Z 1 Acked 1000 Write 8 Done Done Done false
203072024-02-04T14:09:08.020Z 2 Acked 1001 Write 5 Sent Done Done false
203082024-02-04T14:09:08.020Z 3 Acked 1002 Write 6 Sent Done Done false
203092024-02-04T14:09:08.021Z 4 Acked 1003 Read 10 Sent Done Done false
203102024-02-04T14:09:08.021Z 5 Acked 1004 Write 1 Sent Done Done false
203112024-02-04T14:09:08.021Z 6 Acked 1005 Write 1 Sent Done Done false
203122024-02-04T14:09:08.021Z 7 Acked 1006 Read 8 Sent Done Done false
203132024-02-04T14:09:08.021Z 8 Acked 1007 Write 10 Sent Done Done false
203142024-02-04T14:09:08.021Z 9 Acked 1008 Write 8 Sent Done Done false
203152024-02-04T14:09:08.021Z 10 Acked 1009 Write 2 Sent Done Done false
203162024-02-04T14:09:08.021Z 11 Acked 1010 Read 2 Sent Done Done false
203172024-02-04T14:09:08.021Z 12 Acked 1011 Write 8 Sent Done Done false
203182024-02-04T14:09:08.021Z 13 Acked 1012 Read 10 Sent Done Done false
203192024-02-04T14:09:08.021Z 14 Acked 1013 Write 10 Sent Done Done false
203202024-02-04T14:09:08.021Z 15 Acked 1014 Read 4 Sent Done Done false
203212024-02-04T14:09:08.021Z 16 Acked 1015 Write 9 Sent Done Done false
203222024-02-04T14:09:08.022Z 17 Acked 1016 Flush 0 Sent Done Done false
203232024-02-04T14:09:08.022Z 18 Acked 1017 Write 7 Sent Done Done false
203242024-02-04T14:09:08.022Z 19 Acked 1018 Read 9 Sent Done Done false
203252024-02-04T14:09:08.022Z 20 Acked 1019 Read 2 Sent Done Done false
203262024-02-04T14:09:08.022Z 21 Acked 1020 Read 4 Sent Done Done false
203272024-02-04T14:09:08.022Z 22 Acked 1021 Read 6 Sent Done Done false
203282024-02-04T14:09:08.022Z 23 Acked 1022 Read 6 Sent Done Done false
203292024-02-04T14:09:08.022Z 24 Acked 1023 Read 10 Sent Done Done false
203302024-02-04T14:09:08.022Z 25 Acked 1024 Write 5 Sent Done Done false
203312024-02-04T14:09:08.022Z 26 Acked 1025 Read 8 Sent Done Done false
203322024-02-04T14:09:08.022Z 27 Acked 1026 Read 6 Sent Done Done false
203332024-02-04T14:09:08.022Z 28 Acked 1027 Write 7 Sent Done Done false
203342024-02-04T14:09:08.022Z 29 Acked 1028 Write 5 Sent Sent Done false
203352024-02-04T14:09:08.023Z 30 Acked 1029 Write 4 Sent Sent Sent false
203362024-02-04T14:09:08.023Z STATES DS:0 DS:1 DS:2 TOTAL
203372024-02-04T14:09:08.023Z New 0 0 0 0
203382024-02-04T14:09:08.023Z Sent 29 2 1 32
203392024-02-04T14:09:08.023Z Done 1 28 29 58
203402024-02-04T14:09:08.023Z Skipped 0 0 0 0
203412024-02-04T14:09:08.023Z Error 0 0 0 0
203422024-02-04T14:09:08.023ZLast Flush: 0 1016 1016
203432024-02-04T14:09:08.023ZDownstairs last five completed:
203442024-02-04T14:09:08.023ZUpstairs last five completed: 30 29 28 27 26
203452024-02-04T14:09:08.047Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:09:07.051988761Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1884}
203462024-02-04T14:09:08.047ZWrote out file "/tmp/test_repair_verify.data"
203472024-02-04T14:09:08.047Z
203482024-02-04T14:09:08.047ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203492024-02-04T14:09:08.059ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203502024-02-04T14:09:08.059Z 0 000-019 176 176 176 301 301 301 F F F
203512024-02-04T14:09:08.059Z 1 020-039 186 186 186 318 318 318 F F F
203522024-02-04T14:09:08.059Z 2 040-059 179 188 188 307 320 320 T F F <---
203532024-02-04T14:09:08.059Z 3 060-079 178 188 188 304 320 320 T F F <---
203542024-02-04T14:09:08.059Z 4 080-099 184 184 184 315 315 315 F F F
203552024-02-04T14:09:08.059Z 5 100-119 179 188 188 307 320 320 F T T <---
203562024-02-04T14:09:08.059Z 6 120-139 182 188 188 311 320 320 F F F <---
203572024-02-04T14:09:08.059Z 7 140-159 184 188 188 315 320 320 F F F <---
203582024-02-04T14:09:08.063Z 8 160-179 184 184 184 315 315 315 F F F
203592024-02-04T14:09:08.063Z 9 180-199 187 187 187 319 319 319 F F F
203602024-02-04T14:09:08.063Z 10 200-219 187 187 187 319 319 319 F F F
203612024-02-04T14:09:08.063Z 11 220-239 182 188 188 313 320 320 F F F <---
203622024-02-04T14:09:08.063Z 12 240-259 182 188 188 313 320 320 F F F <---
203632024-02-04T14:09:08.063Z 13 260-279 182 188 188 312 320 320 F F F <---
203642024-02-04T14:09:08.063Z 14 280-299 187 188 188 319 320 320 F F F <---
203652024-02-04T14:09:08.064Z 15 300-319 187 188 188 319 320 320 F F F <---
203662024-02-04T14:09:08.064Z 16 320-339 185 185 185 316 316 316 F F F
203672024-02-04T14:09:08.064Z 17 340-359 187 187 187 319 319 319 F F F
203682024-02-04T14:09:08.064Z 18 360-379 178 178 178 306 306 306 F F F
203692024-02-04T14:09:08.064Z 19 380-399 179 179 179 307 307 307 F T T <---
203702024-02-04T14:09:08.064Z 20 400-419 179 179 179 307 307 307 F F F
203712024-02-04T14:09:08.064Z 21 420-439 186 188 188 318 320 320 F T T <---
203722024-02-04T14:09:08.064Z 22 440-459 186 186 186 318 318 318 F T T <---
203732024-02-04T14:09:08.064Z 23 460-479 186 186 186 318 318 318 F F F
203742024-02-04T14:09:08.064Z 24 480-499 187 187 187 319 319 319 F F F
203752024-02-04T14:09:08.064Z 25 500-519 187 188 188 319 320 320 F F F <---
203762024-02-04T14:09:08.064Z 26 520-539 184 184 184 315 315 315 F F F
203772024-02-04T14:09:08.064Z 27 540-559 182 182 182 311 311 311 F F F
203782024-02-04T14:09:08.065Z 28 560-579 178 178 178 305 305 305 F F F
203792024-02-04T14:09:08.065Z 29 580-599 182 182 182 311 311 311 F T T <---
203802024-02-04T14:09:08.065ZMax gen: 188, Max flush: 320
203812024-02-04T14:09:08.065ZError: Difference in extent metadata found!
203822024-02-04T14:09:08.065ZA Difference in extent metadata is expected here
203832024-02-04T14:09:08.065ZOn loop 94
203842024-02-04T14:09:08.065Z
203852024-02-04T14:09:08.069ZVerifying data now
203862024-02-04T14:09:13.342ZNo change after verify
203872024-02-04T14:09:13.342ZLoop: 94 Downstairs dump after verify (and repair):
203882024-02-04T14:09:13.359ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203892024-02-04T14:09:13.359Z 0 000-019 176 176 176 301 301 301 F F F
203902024-02-04T14:09:13.359Z 1 020-039 186 186 186 318 318 318 F F F
203912024-02-04T14:09:13.359Z 2 040-059 188 188 188 320 320 320 F F F
203922024-02-04T14:09:13.359Z 3 060-079 188 188 188 320 320 320 F F F
203932024-02-04T14:09:13.359Z 4 080-099 184 184 184 315 315 315 F F F
203942024-02-04T14:09:13.359Z 5 100-119 189 189 189 321 321 321 F F F
203952024-02-04T14:09:13.359Z 6 120-139 188 188 188 320 320 320 F F F
203962024-02-04T14:09:13.362Z 7 140-159 188 188 188 320 320 320 F F F
203972024-02-04T14:09:13.363Z 8 160-179 184 184 184 315 315 315 F F F
203982024-02-04T14:09:13.363Z 9 180-199 187 187 187 319 319 319 F F F
203992024-02-04T14:09:13.363Z 10 200-219 187 187 187 319 319 319 F F F
204002024-02-04T14:09:13.363Z 11 220-239 188 188 188 320 320 320 F F F
204012024-02-04T14:09:13.363Z 12 240-259 188 188 188 320 320 320 F F F
204022024-02-04T14:09:13.363Z 13 260-279 188 188 188 320 320 320 F F F
204032024-02-04T14:09:13.363Z 14 280-299 188 188 188 320 320 320 F F F
204042024-02-04T14:09:13.363Z 15 300-319 188 188 188 320 320 320 F F F
204052024-02-04T14:09:13.363Z 16 320-339 185 185 185 316 316 316 F F F
204062024-02-04T14:09:13.363Z 17 340-359 187 187 187 319 319 319 F F F
204072024-02-04T14:09:13.363Z 18 360-379 178 178 178 306 306 306 F F F
204082024-02-04T14:09:13.363Z 19 380-399 189 189 189 321 321 321 F F F
204092024-02-04T14:09:13.363Z 20 400-419 179 179 179 307 307 307 F F F
204102024-02-04T14:09:13.363Z 21 420-439 189 189 189 321 321 321 F F F
204112024-02-04T14:09:13.363Z 22 440-459 189 189 189 321 321 321 F F F
204122024-02-04T14:09:13.363Z 23 460-479 186 186 186 318 318 318 F F F
204132024-02-04T14:09:13.363Z 24 480-499 187 187 187 319 319 319 F F F
204142024-02-04T14:09:13.363Z 25 500-519 188 188 188 320 320 320 F F F
204152024-02-04T14:09:13.363Z 26 520-539 184 184 184 315 315 315 F F F
204162024-02-04T14:09:13.363Z 27 540-559 182 182 182 311 311 311 F F F
204172024-02-04T14:09:13.363Z 28 560-579 178 178 178 305 305 305 F F F
204182024-02-04T14:09:13.363Z 29 580-599 189 189 189 321 321 321 F F F
204192024-02-04T14:09:13.363ZMax gen: 189, Max flush: 321
204202024-02-04T14:09:13.367Z
204212024-02-04T14:09:13.367ZBegin loop 95
204222024-02-04T14:09:13.367ZDownstairs to restart: 0
204232024-02-04T14:09:13.393Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.425642264Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892}
204242024-02-04T14:09:13.393Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.426164877Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892}
204252024-02-04T14:09:13.398Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.426214528Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892}
204262024-02-04T14:09:13.398Z{"msg":"Crucible 91f138ff-1c83-4b62-8719-1a3ee17d4d02 has session id: b3779326-66ca-4d0b-a981-0159eeafe1b4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.426291421Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204272024-02-04T14:09:13.398Z{"msg":"Upstairs opts: Upstairs UUID: 91f138ff-1c83-4b62-8719-1a3ee17d4d02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.426345351Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204282024-02-04T14:09:13.398Z{"msg":"Crucible stats registered with UUID: 91f138ff-1c83-4b62-8719-1a3ee17d4d02","v":0,"name":"crucible","level":30Crucible runtime is spawned
204292024-02-04T14:09:13.398Z,"time":"2024-02-04T14:09:12.426454175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204302024-02-04T14:09:13.398Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.426607237Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892}
204312024-02-04T14:09:13.398Z{"msg":"91f138ff-1c83-4b62-8719-1a3ee17d4d02 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.426786787Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204322024-02-04T14:09:13.398Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.42682865Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204332024-02-04T14:09:13.398Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.426881656Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204342024-02-04T14:09:13.398Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.426973625Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204352024-02-04T14:09:13.398Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.427016826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204362024-02-04T14:09:13.398Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.427097221Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204372024-02-04T14:09:13.398Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.427154142Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204382024-02-04T14:09:13.398Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.42738554Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204392024-02-04T14:09:13.398Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.427495666Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204402024-02-04T14:09:13.398Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.427593838Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204412024-02-04T14:09:13.398Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.427638433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204422024-02-04T14:09:13.398Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.427681537Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204432024-02-04T14:09:13.399Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.427729687Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204442024-02-04T14:09:13.399Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.428478305Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204452024-02-04T14:09:13.399Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.428544324Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204462024-02-04T14:09:13.399Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.428737338Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204472024-02-04T14:09:13.399Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.428838867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204482024-02-04T14:09:13.399Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.428885242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204492024-02-04T14:09:13.399Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.428980457Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204502024-02-04T14:09:13.404Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.4296778Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204512024-02-04T14:09:13.404Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.429744895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204522024-02-04T14:09:13.405Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.429790402Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204532024-02-04T14:09:13.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.430574713Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204542024-02-04T14:09:13.405Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.430641535Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204552024-02-04T14:09:13.405Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.430668236Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204562024-02-04T14:09:13.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.430709127Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204572024-02-04T14:09:13.405Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.430733883Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204582024-02-04T14:09:13.405Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.430758052Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204592024-02-04T14:09:13.405Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.43079783Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204602024-02-04T14:09:13.405Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.430822441Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204612024-02-04T14:09:13.405Z{"msg":"[0]R flush_numbers[0..12]: [301, 318, 320, 320, 315, 321, 320, 320, 315, 319, 319, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.430872397Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204622024-02-04T14:09:13.405Z{"msg":"[0]R generation[0..12]: [176, 186, 188, 188, 184, 189, 188, 188, 184, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.43089771Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
204632024-02-04T14:09:13.405Z,"":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204642024-02-04T14:09:13.405Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431124706Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204652024-02-04T14:09:13.405Z{"msg":"[1]R flush_numbers[0..12]: [301, 318, 320, 320, 315, 321, 320, 320, 315, 319, 319, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431162807Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204662024-02-04T14:09:13.405Z{"msg":"[1]R generation[0..12]: [176, 186, 188, 188, 184, 189, 188, 188, 184, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431187956Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204672024-02-04T14:09:13.405Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431212069Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204682024-02-04T14:09:13.405Z{"msg":"[2]R flush_numbers[0..12]: [301, 318, 320, 320, 315, 321, 320, 320, 315, 319, 319, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.43126307Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204692024-02-04T14:09:13.405Z{"msg":"[2]R generation[0..12]: [176, 186, 188, 188, 184, 189, 188, 188, 184, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431288268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204702024-02-04T14:09:13.405Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431335436Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204712024-02-04T14:09:13.405Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431377423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204722024-02-04T14:09:13.405Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431401467Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204732024-02-04T14:09:13.405Z{"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431446027Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204742024-02-04T14:09:13.406Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431469174Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204752024-02-04T14:09:13.406Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431510126Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204762024-02-04T14:09:13.406Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431532038Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204772024-02-04T14:09:13.406Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431556124Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204782024-02-04T14:09:13.406Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431578523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204792024-02-04T14:09:13.406Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431600826Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204802024-02-04T14:09:13.406Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431651311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204812024-02-04T14:09:13.406Z{"msg":"91f138ff-1c83-4b62-8719-1a3ee17d4d02 is now active with session: b3779326-66ca-4d0b-a981-0159eeafe1b4","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431674317Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204822024-02-04T14:09:13.406Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431721114Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892,"session_id":"b3779326-66ca-4d0b-a981-0159eeafe1b4"}
204832024-02-04T14:09:13.406Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:12.431742895Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892}
204842024-02-04T14:09:13.406ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204852024-02-04T14:09:13.406ZLoading write count information from file "/tmp/test_repair_verify.data"
204862024-02-04T14:09:13.406ZRun Repair workload
204872024-02-04T14:09:13.406Z01/30 Write block 123 len 28672 data: 14 15 17 17 15 12 9
204882024-02-04T14:09:13.410Z02/30 Read block 331 len 4096
204892024-02-04T14:09:13.410Z03/30 Read block 136 len 32768
204902024-02-04T14:09:13.410Z04/30 Write block 37 len 40960 data: 19 16 15 13 14 16 14 14 15 15
204912024-02-04T14:09:13.410Z05/30 Read block 35 len 40960
204922024-02-04T14:09:13.413Z06/30 Read block 362 len 12288
204932024-02-04T14:09:13.413Z07/30 Read block 473 len 32768
204942024-02-04T14:09:13.416Z08/30 Write block 424 len 36864 data: 8 10 9 11 12 12 15 16 15
204952024-02-04T14:09:13.416Z09/30 Read block 374 len 20480
204962024-02-04T14:09:13.419Z10/30 Read block 340 len 4096
204972024-02-04T14:09:13.419Z11/30 Write block 23 len 12288 data: 16 14 12
204982024-02-04T14:09:13.420Z12/30 Write block 500 len 36864 data: 19 18 14 14 15 13 18 18 18
204992024-02-04T14:09:13.422Z13/30 Read block 548 len 8192
205002024-02-04T14:09:13.423Z14/30 Read block 361 len 40960
205012024-02-04T14:09:13.426Z15/30 Write block 240 len 4096 data: 20
205022024-02-04T14:09:13.426Z16/30 Write block 477 len 4096 data: 11
205032024-02-04T14:09:13.426Z17/30 Flush
205042024-02-04T14:09:13.429Z18/30 Write block 76 len 20480 data: 12 15 14 13 16
205052024-02-04T14:09:13.432Z19/30 Write block 476 len 16384 data: 11 12 14 14
205062024-02-04T14:09:13.432Z20/30 Read block 467 len 16384
205072024-02-04T14:09:13.437Z21/30 Write block 535 len 36864 data: 9 11 8 11 12 14 15 14 17
205082024-02-04T14:09:13.438Z22/30 Read block 388 len 24576
205092024-02-04T14:09:13.440Z23/30 Write block 471 len 36864 data: 13 13 12 11 13 12 13 15 15
205102024-02-04T14:09:13.440Z24/30 Write block 386 len 28672 data: 9 11 9 10 8 13 14
205112024-02-04T14:09:13.441Z25/30 Write block 493 len 24576 data: 17 14 14 16 15 17
205122024-02-04T14:09:13.441Z26/30 Read block 225 len 32768
205132024-02-04T14:09:13.443Z27/30 Read block 223 len 8192
205142024-02-04T14:09:13.443Z28/30 Read block 246 len 8192
205152024-02-04T14:09:13.446Z29/30 Read block 17 len 40960
205162024-02-04T14:09:13.446Z30/30 Read block 403 len 28672
205172024-02-04T14:09:13.450Z----------------------------------------------------------------
205182024-02-04T14:09:13.450Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
205192024-02-04T14:09:13.450ZStates: Active Active Active
205202024-02-04T14:09:13.450ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
205212024-02-04T14:09:13.450Z 1 Acked 1000 Write 7 Done Done Done false
205222024-02-04T14:09:13.450Z 2 Acked 1001 Read 1 Done Done Done false
205232024-02-04T14:09:13.450Z 3 Acked 1002 Read 8 Sent Done Done false
205242024-02-04T14:09:13.450Z 4 Acked 1003 Write 10 Sent Done Done false
205252024-02-04T14:09:13.450Z 5 Acked 1004 Read 10 Sent Done Done false
205262024-02-04T14:09:13.450Z 6 Acked 1005 Read 3 Sent Done Done false
205272024-02-04T14:09:13.450Z 7 Acked 1006 Read 8 Sent Done Done false
205282024-02-04T14:09:13.450Z 8 Acked 1007 Write 9 Sent Done Done false
205292024-02-04T14:09:13.451Z 9 Acked 1008 Read 5 Sent Done Done false
205302024-02-04T14:09:13.451Z 10 Acked 1009 Read 1 Sent Done Done false
205312024-02-04T14:09:13.451Z 11 Acked 1010 Write 3 Sent Done Done false
205322024-02-04T14:09:13.451Z 12 Acked 1011 Write 9 Sent Done Done false
205332024-02-04T14:09:13.451Z 13 Acked 1012 Read 2 Sent Done Done false
205342024-02-04T14:09:13.451Z 14 Acked 1013 Read 10 Sent Done Done false
205352024-02-04T14:09:13.451Z 15 Acked 1014 Write 1 Sent Done Done false
205362024-02-04T14:09:13.451Z 16 Acked 1015 Write 1 Sent Done Done false
205372024-02-04T14:09:13.451Z 17 Acked 1016 Flush 0 Sent Done Done false
205382024-02-04T14:09:13.451Z 18 Acked 1017 Write 5 Sent Done Done false
205392024-02-04T14:09:13.451Z 19 Acked 1018 Write 4 Sent Done Done false
205402024-02-04T14:09:13.451Z 20 Acked 1019 Read 4 Sent Done Done false
205412024-02-04T14:09:13.451Z 21 Acked 1020 Write 9 Sent Done Done false
205422024-02-04T14:09:13.451Z 22 Acked 1021 Read 6 Sent Done Done false
205432024-02-04T14:09:13.451Z 23 Acked 1022 Write 9 Sent Done Done false
205442024-02-04T14:09:13.451Z 24 Acked 1023 Write 7 Sent Done Done false
205452024-02-04T14:09:13.451Z 25 Acked 1024 Write 6 Sent Done Done false
205462024-02-04T14:09:13.451Z 26 Acked 1025 Read 8 Sent Done Done false
205472024-02-04T14:09:13.451Z 27 Acked 1026 Read 2 Sent Done Done false
205482024-02-04T14:09:13.451Z 28 Acked 1027 Read 2 Sent Done Done false
205492024-02-04T14:09:13.452Z 29 Acked 1028 Read 10 Sent Done Done false
205502024-02-04T14:09:13.452Z 30 Acked 1029 Read 7 Sent Done Done false
205512024-02-04T14:09:13.452Z STATES DS:0 DS:1 DS:2 TOTAL
205522024-02-04T14:09:13.452Z New 0 0 0 0
205532024-02-04T14:09:13.452Z Sent 28 0 0 28
205542024-02-04T14:09:13.452Z Done 2 30 30 62
205552024-02-04T14:09:13.452Z Skipped 0 0 0 0
205562024-02-04T14:09:13.452Z Error 0 0 0 0
205572024-02-04T14:09:13.452ZLast Flush: 0 1016 1016
205582024-02-04T14:09:13.452ZDownstairs last five completed:
205592024-02-04T14:09:13.452ZUpstairs last five completed: 30 29 28 27 26
205602024-02-04T14:09:13.452Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:09:12.480687242Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1892}
205612024-02-04T14:09:13.452ZWrote out file "/tmp/test_repair_verify.data"
205622024-02-04T14:09:13.455Z
205632024-02-04T14:09:13.459ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205642024-02-04T14:09:13.485ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205652024-02-04T14:09:13.485Z 0 000-019 176 176 176 301 301 301 F F F
205662024-02-04T14:09:13.485Z 1 020-039 186 190 190 318 322 322 F F F <---
205672024-02-04T14:09:13.486Z 2 040-059 188 190 190 320 322 322 F F F <---
205682024-02-04T14:09:13.486Z 3 060-079 188 188 188 320 320 320 F T T <---
205692024-02-04T14:09:13.486Z 4 080-099 184 184 184 315 315 315 F T T <---
205702024-02-04T14:09:13.486Z 5 100-119 189 189 189 321 321 321 F F F
205712024-02-04T14:09:13.489Z 6 120-139 188 190 190 320 322 322 T F F <---
205722024-02-04T14:09:13.489Z 7 140-159 188 188 188 320 320 320 F F F
205732024-02-04T14:09:13.489Z 8 160-179 184 184 184 315 315 315 F F F
205742024-02-04T14:09:13.489Z 9 180-199 187 187 187 319 319 319 F F F
205752024-02-04T14:09:13.489Z 10 200-219 187 187 187 319 319 319 F F F
205762024-02-04T14:09:13.489Z 11 220-239 188 188 188 320 320 320 F F F
205772024-02-04T14:09:13.489Z 12 240-259 188 190 190 320 322 322 F F F <---
205782024-02-04T14:09:13.489Z 13 260-279 188 188 188 320 320 320 F F F
205792024-02-04T14:09:13.489Z 14 280-299 188 188 188 320 320 320 F F F
205802024-02-04T14:09:13.489Z 15 300-319 188 188 188 320 320 320 F F F
205812024-02-04T14:09:13.489Z 16 320-339 185 185 185 316 316 316 F F F
205822024-02-04T14:09:13.489Z 17 340-359 187 187 187 319 319 319 F F F
205832024-02-04T14:09:13.489Z 18 360-379 178 178 178 306 306 306 F F F
205842024-02-04T14:09:13.490Z 19 380-399 189 189 189 321 321 321 F T T <---
205852024-02-04T14:09:13.490Z 20 400-419 179 179 179 307 307 307 F F F
205862024-02-04T14:09:13.490Z 21 420-439 189 190 190 321 322 322 F F F <---
205872024-02-04T14:09:13.490Z 22 440-459 189 189 189 321 321 321 F F F
205882024-02-04T14:09:13.490Z 23 460-479 186 190 190 318 322 322 F T T <---
205892024-02-04T14:09:13.490Z 24 480-499 187 187 187 319 319 319 F T T <---
205902024-02-04T14:09:13.490Z 25 500-519 188 190 190 320 322 322 F F F <---
205912024-02-04T14:09:13.490Z 26 520-539 184 184 184 315 315 315 F T T <---
205922024-02-04T14:09:13.490Z 27 540-559 182 182 182 311 311 311 F T T <---
205932024-02-04T14:09:13.490Z 28 560-579 178 178 178 305 305 305 F F F
205942024-02-04T14:09:13.490Z 29 580-599 189 189 189 321 321 321 F F F
205952024-02-04T14:09:13.490ZMax gen: 190, Max flush: 322
205962024-02-04T14:09:13.490ZError: Difference in extent metadata found!
205972024-02-04T14:09:13.493ZA Difference in extent metadata is expected here
205982024-02-04T14:09:13.493ZOn loop 95
205992024-02-04T14:09:13.493Z
206002024-02-04T14:09:13.496ZVerifying data now
206012024-02-04T14:09:18.653ZNo change after verify
206022024-02-04T14:09:18.653ZLoop: 95 Downstairs dump after verify (and repair):
206032024-02-04T14:09:18.675ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206042024-02-04T14:09:18.675Z 0 000-019 176 176 176 301 301 301 F F F
206052024-02-04T14:09:18.675Z 1 020-039 190 190 190 322 322 322 F F F
206062024-02-04T14:09:18.675Z 2 040-059 190 190 190 322 322 322 F F F
206072024-02-04T14:09:18.675Z 3 060-079 191 191 191 323 323 323 F F F
206082024-02-04T14:09:18.675Z 4 080-099 191 191 191 323 323 323 F F F
206092024-02-04T14:09:18.675Z 5 100-119 189 189 189 321 321 321 F F F
206102024-02-04T14:09:18.675Z 6 120-139 190 190 190 322 322 322 F F F
206112024-02-04T14:09:18.675Z 7 140-159 188 188 188 320 320 320 F F F
206122024-02-04T14:09:18.675Z 8 160-179 184 184 184 315 315 315 F F F
206132024-02-04T14:09:18.679Z 9 180-199 187 187 187 319 319 319 F F F
206142024-02-04T14:09:18.679Z 10 200-219 187 187 187 319 319 319 F F F
206152024-02-04T14:09:18.679Z 11 220-239 188 188 188 320 320 320 F F F
206162024-02-04T14:09:18.679Z 12 240-259 190 190 190 322 322 322 F F F
206172024-02-04T14:09:18.679Z 13 260-279 188 188 188 320 320 320 F F F
206182024-02-04T14:09:18.679Z 14 280-299 188 188 188 320 320 320 F F F
206192024-02-04T14:09:18.679Z 15 300-319 188 188 188 320 320 320 F F F
206202024-02-04T14:09:18.679Z 16 320-339 185 185 185 316 316 316 F F F
206212024-02-04T14:09:18.679Z 17 340-359 187 187 187 319 319 319 F F F
206222024-02-04T14:09:18.679Z 18 360-379 178 178 178 306 306 306 F F F
206232024-02-04T14:09:18.679Z 19 380-399 191 191 191 323 323 323 F F F
206242024-02-04T14:09:18.679Z 20 400-419 179 179 179 307 307 307 F F F
206252024-02-04T14:09:18.679Z 21 420-439 190 190 190 322 322 322 F F F
206262024-02-04T14:09:18.679Z 22 440-459 189 189 189 321 321 321 F F F
206272024-02-04T14:09:18.679Z 23 460-479 191 191 191 323 323 323 F F F
206282024-02-04T14:09:18.679Z 24 480-499 191 191 191 323 323 323 F F F
206292024-02-04T14:09:18.679Z 25 500-519 190 190 190 322 322 322 F F F
206302024-02-04T14:09:18.680Z 26 520-539 191 191 191 323 323 323 F F F
206312024-02-04T14:09:18.680Z 27 540-559 191 191 191 323 323 323 F F F
206322024-02-04T14:09:18.680Z 28 560-579 178 178 178 305 305 305 F F F
206332024-02-04T14:09:18.680Z 29 580-599 189 189 189 321 321 321 F F F
206342024-02-04T14:09:18.680ZMax gen: 191, Max flush: 323
206352024-02-04T14:09:18.682Z
206362024-02-04T14:09:18.683ZBegin loop 96
206372024-02-04T14:09:18.683ZDownstairs to restart: 1
206382024-02-04T14:09:18.709Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.741792587Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900}
206392024-02-04T14:09:18.714Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"Crucible runtime is spawned
206402024-02-04T14:09:18.714Z93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.742534103Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900}
206412024-02-04T14:09:18.714Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.742600977Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900}
206422024-02-04T14:09:18.714Z{"msg":"Crucible 2ffe1915-a3fe-4f6f-bebe-1a7d2db6b4b6 has session id: 26969302-2d38-4a62-8919-019be90c074d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.742626923Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206432024-02-04T14:09:18.714Z{"msg":"Upstairs opts: Upstairs UUID: 2ffe1915-a3fe-4f6f-bebe-1a7d2db6b4b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.742678022Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206442024-02-04T14:09:18.714Z{"msg":"Crucible stats registered with UUID: 2ffe1915-a3fe-4f6f-bebe-1a7d2db6b4b6","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.742702171Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206452024-02-04T14:09:18.714Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.742863496Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900}
206462024-02-04T14:09:18.714Z{"msg":"2ffe1915-a3fe-4f6f-bebe-1a7d2db6b4b6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.743137917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206472024-02-04T14:09:18.714Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.743240149Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206482024-02-04T14:09:18.714Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.743284168Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206492024-02-04T14:09:18.714Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.74337481Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206502024-02-04T14:09:18.714Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.743426165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206512024-02-04T14:09:18.715Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.743518604Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206522024-02-04T14:09:18.715Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.743608842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206532024-02-04T14:09:18.715Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.743687718Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206542024-02-04T14:09:18.715Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.743819322Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206552024-02-04T14:09:18.715Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.74397547Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206562024-02-04T14:09:18.715Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.7440383Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206572024-02-04T14:09:18.715Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.744086448Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206582024-02-04T14:09:18.715Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.744173876Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206592024-02-04T14:09:18.715Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.744662627Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206602024-02-04T14:09:18.715Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.74474862Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206612024-02-04T14:09:18.715Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.744798501Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206622024-02-04T14:09:18.715Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.744908316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206632024-02-04T14:09:18.715Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.745100216Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206642024-02-04T14:09:18.715Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.745178816Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206652024-02-04T14:09:18.720Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.745585917Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206662024-02-04T14:09:18.720Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.745801958Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206672024-02-04T14:09:18.720Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.746424345Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206682024-02-04T14:09:18.720Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.746669631Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206692024-02-04T14:09:18.720Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.746735953Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206702024-02-04T14:09:18.720Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.746761278Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206712024-02-04T14:09:18.720Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.746781976Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206722024-02-04T14:09:18.720Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.746804747Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206732024-02-04T14:09:18.720Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.746836275Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206742024-02-04T14:09:18.720Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747166218Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206752024-02-04T14:09:18.720Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747226266Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206762024-02-04T14:09:18.720Z{"msg":"[0]R flush_numbers[0..12]: [301, 322, 322, 323, 323, 321, 322, 320, 315, 319, 319, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.74725279Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206772024-02-04T14:09:18.720Z{"msg":"[0]R generation[0..12]: [176, 190, 190, 191, 191, 189, 190, 188, 184, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747276912Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206782024-02-04T14:09:18.720Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747301083Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206792024-02-04T14:09:18.720Z{"msg":"[1]R flush_numbers[0..12]: [301, 322, 322, 323, 323, 321, 322, 320, 315, 319, 319, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747324601Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206802024-02-04T14:09:18.720Z{"msg":"[1]R generation[0..12]: [176, 190, 190, 191, 191, 189, 190, 188, 184, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.74736748Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206812024-02-04T14:09:18.720Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747392968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206822024-02-04T14:09:18.720Z{"msg":"[2]R flush_numbers[0..12]: [301, 322, 322, 323, 323, 321, 322, 320, 315, 319, 319, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747416641Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206832024-02-04T14:09:18.720Z{"msg":"[2]R generation[0..12]: [176, 190, 190, 191, 191, 189, 190, 188, 184, 187, 187, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747458721Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206842024-02-04T14:09:18.721Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"Wait for a query_work_queue command to finish before sending IO
206852024-02-04T14:09:18.721Zname":"crucible","level":30,"time":"2024-02-04T14:09:17.747487298Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206862024-02-04T14:09:18.721Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747510049Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206872024-02-04T14:09:18.721Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.74756049Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206882024-02-04T14:09:18.721Z{"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747586968Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206892024-02-04T14:09:18.721Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747629612Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206902024-02-04T14:09:18.721Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747653703Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206912024-02-04T14:09:18.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747707569Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206922024-02-04T14:09:18.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.74773138Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206932024-02-04T14:09:18.721Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747753425Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206942024-02-04T14:09:18.721Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747781121Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206952024-02-04T14:09:18.721Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747805141Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206962024-02-04T14:09:18.721Z{"msg":"2ffe1915-a3fe-4f6f-bebe-1a7d2db6b4b6 is now active with session: 26969302-2d38-4a62-8919-019be90c074d","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747829084Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206972024-02-04T14:09:18.721Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747882041Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900,"session_id":"26969302-2d38-4a62-8919-019be90c074d"}
206982024-02-04T14:09:18.721Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:17.747904478Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900}
206992024-02-04T14:09:18.721ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
207002024-02-04T14:09:18.721ZLoading write count information from file "/tmp/test_repair_verify.data"
207012024-02-04T14:09:18.721ZRun Repair workload
207022024-02-04T14:09:18.721Z01/30 Write block 38 len 24576 data: 17 16 14 15 17 15
207032024-02-04T14:09:18.725Z02/30 Write block 420 len 36864 data: 14 12 13 13 9 11 10 12 13
207042024-02-04T14:09:18.725Z03/30 Write block 73 len 28672 data: 12 12 11 13 16 15 14
207052024-02-04T14:09:18.725Z04/30 Write block 495 len 12288 data: 15 17 16
207062024-02-04T14:09:18.725Z05/30 Read block 100 len 40960
207072024-02-04T14:09:18.728Z06/30 Write block 465 len 12288 data: 20 23 20
207082024-02-04T14:09:18.728Z07/30 Write block 111 len 36864 data: 17 21 18 19 19 16 16 13 14
207092024-02-04T14:09:18.731Z08/30 Read block 542 len 16384
207102024-02-04T14:09:18.731Z09/30 Write block 228 len 20480 data: 15 13 13 13 13
207112024-02-04T14:09:18.734Z10/30 Read block 510 len 8192
207122024-02-04T14:09:18.734Z11/30 Write block 576 len 28672 data: 14 15 15 16 18 17 14
207132024-02-04T14:09:18.734Z12/30 Read block 337 len 12288
207142024-02-04T14:09:18.739Z13/30 Write block 308 len 28672 data: 15 15 17 16 12 13 13
207152024-02-04T14:09:18.740Z14/30 Read block 384 len 4096
207162024-02-04T14:09:18.743Z15/30 Write block 125 len 16384 data: 18 18 16 13
207172024-02-04T14:09:18.743Z16/30 Read block 49 len 36864
207182024-02-04T14:09:18.743Z17/30 Read block 464 len 20480
207192024-02-04T14:09:18.743Z18/30 Read block 254 len 28672
207202024-02-04T14:09:18.745Z19/30 Flush
207212024-02-04T14:09:18.750Z20/30 Write block 450 len 20480 data: 16 13 14 13 15
207222024-02-04T14:09:18.753Z21/30 Write block 569 len 36864 data: 12 15 13 13 12 12 13 15 16
207232024-02-04T14:09:18.753Z22/30 Write block 439 len 16384 data: 11 15 14 15
207242024-02-04T14:09:18.756Z23/30 Read block 285 len 36864
207252024-02-04T14:09:18.756Z24/30 Write block 393 len 8192 data: 16 15
207262024-02-04T14:09:18.756Z25/30 Read block 317 len 16384
207272024-02-04T14:09:18.759Z26/30 Write block 63 len 12288 data: 15 12 12
207282024-02-04T14:09:18.759Z27/30 Write block 88 len 28672 data: 11 7 7 9 10 10 12
207292024-02-04T14:09:18.759Z28/30 Read block 346 len 16384
207302024-02-04T14:09:18.762Z29/30 Read block 489 len 20480
207312024-02-04T14:09:18.762Z30/30 Read block 23 len 12288
207322024-02-04T14:09:18.765Z----------------------------------------------------------------
207332024-02-04T14:09:18.765Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
207342024-02-04T14:09:18.765ZStates: Active Active Active
207352024-02-04T14:09:18.765ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
207362024-02-04T14:09:18.766Z 1 Acked 1000 Write 6 Done Done Done false
207372024-02-04T14:09:18.766Z 2 Acked 1001 Write 9 Done Done Done false
207382024-02-04T14:09:18.766Z 3 Acked 1002 Write 7 Done Done Done false
207392024-02-04T14:09:18.766Z 4 Acked 1003 Write 3 Done Done Done false
207402024-02-04T14:09:18.766Z 5 Acked 1004 Read 10 Done Done Done false
207412024-02-04T14:09:18.766Z 6 Acked 1005 Write 3 Done Sent Done false
207422024-02-04T14:09:18.766Z 7 Acked 1006 Write 9 Done Sent Done false
207432024-02-04T14:09:18.766Z 8 Acked 1007 Read 4 Done Sent Done false
207442024-02-04T14:09:18.766Z 9 Acked 1008 Write 5 Done Sent Done false
207452024-02-04T14:09:18.766Z 10 Acked 1009 Read 2 Done Sent Done false
207462024-02-04T14:09:18.766Z 11 Acked 1010 Write 7 Done Sent Done false
207472024-02-04T14:09:18.766Z 12 Acked 1011 Read 3 Done Sent Done false
207482024-02-04T14:09:18.766Z 13 Acked 1012 Write 7 Done Sent Done false
207492024-02-04T14:09:18.766Z 14 Acked 1013 Read 1 Done Sent Done false
207502024-02-04T14:09:18.766Z 15 Acked 1014 Write 4 Done Sent Done false
207512024-02-04T14:09:18.766Z 16 Acked 1015 Read 9 Done Sent Done false
207522024-02-04T14:09:18.766Z 17 Acked 1016 Read 5 Done Sent Done false
207532024-02-04T14:09:18.766Z 18 Acked 1017 Read 7 Done Sent Done false
207542024-02-04T14:09:18.766Z 19 Acked 1018 Flush 0 Done Sent Done false
207552024-02-04T14:09:18.766Z 20 Acked 1019 Write 5 Done Sent Done false
207562024-02-04T14:09:18.766Z 21 Acked 1020 Write 9 Done Sent Done false
207572024-02-04T14:09:18.766Z 22 Acked 1021 Write 4 Done Sent Done false
207582024-02-04T14:09:18.766Z 23 Acked 1022 Read 9 Done Sent Done false
207592024-02-04T14:09:18.766Z 24 Acked 1023 Write 2 Done Sent Done false
207602024-02-04T14:09:18.767Z 25 Acked 1024 Read 4 Done Sent Done false
207612024-02-04T14:09:18.767Z 26 Acked 1025 Write 3 Done Sent Done false
207622024-02-04T14:09:18.767Z 27 Acked 1026 Write 7 Done Sent Done false
207632024-02-04T14:09:18.767Z 28 Acked 1027 Read 4 Done Sent Done false
207642024-02-04T14:09:18.767Z 29 Acked 1028 Read 5 Done Sent Done false
207652024-02-04T14:09:18.767Z 30 Acked 1029 Read 3 Done Sent Done false
207662024-02-04T14:09:18.767Z STATES DS:0 DS:1 DS:2 TOTAL
207672024-02-04T14:09:18.767Z New 0 0 0 0
207682024-02-04T14:09:18.767Z Sent 0 25 0 25
207692024-02-04T14:09:18.767Z Done 30 5 30 65
207702024-02-04T14:09:18.767Z Skipped 0 0 0 0
207712024-02-04T14:09:18.767Z Error 0 0 0 0
207722024-02-04T14:09:18.767ZLast Flush: 1018 0 1018
207732024-02-04T14:09:18.767ZDownstairs last five completed:
207742024-02-04T14:09:18.767ZUpstairs last five completed: 30 29 28 27 26
207752024-02-04T14:09:18.767Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:09:17.796247863Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1900}
207762024-02-04T14:09:18.767ZWrote out file "/tmp/test_repair_verify.data"
207772024-02-04T14:09:18.771Z
207782024-02-04T14:09:18.777ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207792024-02-04T14:09:18.805ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207802024-02-04T14:09:18.805Z 0 000-019 176 176 176 301 301 301 F F F
207812024-02-04T14:09:18.805Z 1 020-039 192 190 192 324 322 324 F T F <---
207822024-02-04T14:09:18.805Z 2 040-059 192 190 192 324 322 324 F T F <---
207832024-02-04T14:09:18.805Z 3 060-079 192 191 192 324 323 324 T T T <---
207842024-02-04T14:09:18.805Z 4 080-099 191 191 191 323 323 323 T F T <---
207852024-02-04T14:09:18.805Z 5 100-119 192 189 192 324 321 324 F F F <---
207862024-02-04T14:09:18.805Z 6 120-139 192 190 192 324 322 324 F F F <---
207872024-02-04T14:09:18.808Z 7 140-159 188 188 188 320 320 320 F F F
207882024-02-04T14:09:18.808Z 8 160-179 184 184 184 315 315 315 F F F
207892024-02-04T14:09:18.808Z 9 180-199 187 187 187 319 319 319 F F F
207902024-02-04T14:09:18.808Z 10 200-219 187 187 187 319 319 319 F F F
207912024-02-04T14:09:18.808Z 11 220-239 192 188 192 324 320 324 F F F <---
207922024-02-04T14:09:18.808Z 12 240-259 190 190 190 322 322 322 F F F
207932024-02-04T14:09:18.808Z 13 260-279 188 188 188 320 320 320 F F F
207942024-02-04T14:09:18.808Z 14 280-299 188 188 188 320 320 320 F F F
207952024-02-04T14:09:18.808Z 15 300-319 192 188 192 324 320 324 F F F <---
207962024-02-04T14:09:18.809Z 16 320-339 185 185 185 316 316 316 F F F
207972024-02-04T14:09:18.809Z 17 340-359 187 187 187 319 319 319 F F F
207982024-02-04T14:09:18.809Z 18 360-379 178 178 178 306 306 306 F F F
207992024-02-04T14:09:18.809Z 19 380-399 191 191 191 323 323 323 T F T <---
208002024-02-04T14:09:18.809Z 20 400-419 179 179 179 307 307 307 F F F
208012024-02-04T14:09:18.809Z 21 420-439 192 190 192 324 322 324 T T T <---
208022024-02-04T14:09:18.809Z 22 440-459 189 189 189 321 321 321 T F T <---
208032024-02-04T14:09:18.809Z 23 460-479 192 191 192 324 323 324 F F F <---
208042024-02-04T14:09:18.809Z 24 480-499 192 191 192 324 323 324 F T F <---
208052024-02-04T14:09:18.809Z 25 500-519 190 190 190 322 322 322 F F F
208062024-02-04T14:09:18.809Z 26 520-539 191 191 191 323 323 323 F F F
208072024-02-04T14:09:18.809Z 27 540-559 191 191 191 323 323 323 F F F
208082024-02-04T14:09:18.809Z 28 560-579 192 178 192 324 305 324 T F T <---
208092024-02-04T14:09:18.809Z 29 580-599 192 189 192 324 321 324 F F F <---
208102024-02-04T14:09:18.809ZMax gen: 192, Max flush: 324
208112024-02-04T14:09:18.809ZError: Difference in extent metadata found!
208122024-02-04T14:09:18.812ZA Difference in extent metadata is expected here
208132024-02-04T14:09:18.812ZOn loop 96
208142024-02-04T14:09:18.812Z
208152024-02-04T14:09:18.815ZVerifying data now
208162024-02-04T14:09:25.102ZNo change after verify
208172024-02-04T14:09:25.102ZLoop: 96 Downstairs dump after verify (and repair):
208182024-02-04T14:09:25.125ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208192024-02-04T14:09:25.126Z 0 000-019 176 176 176 301 301 301 F F F
208202024-02-04T14:09:25.126Z 1 020-039 192 192 192 324 324 324 F F F
208212024-02-04T14:09:25.126Z 2 040-059 192 192 192 324 324 324 F F F
208222024-02-04T14:09:25.126Z 3 060-079 193 193 193 325 325 325 F F F
208232024-02-04T14:09:25.126Z 4 080-099 193 193 193 325 325 325 F F F
208242024-02-04T14:09:25.126Z 5 100-119 192 192 192 324 324 324 F F F
208252024-02-04T14:09:25.126Z 6 120-139 192 192 192 324 324 324 F F F
208262024-02-04T14:09:25.126Z 7 140-159 188 188 188 320 320 320 F F F
208272024-02-04T14:09:25.126Z 8 160-179 184 184 184 315 315 315 F F F
208282024-02-04T14:09:25.126Z 9 180-199 187 187 187 319 319 319 F F F
208292024-02-04T14:09:25.126Z 10 200-219 187 187 187 319 319 319 F F F
208302024-02-04T14:09:25.130Z 11 220-239 192 192 192 324 324 324 F F F
208312024-02-04T14:09:25.130Z 12 240-259 190 190 190 322 322 322 F F F
208322024-02-04T14:09:25.130Z 13 260-279 188 188 188 320 320 320 F F F
208332024-02-04T14:09:25.130Z 14 280-299 188 188 188 320 320 320 F F F
208342024-02-04T14:09:25.130Z 15 300-319 192 192 192 324 324 324 F F F
208352024-02-04T14:09:25.130Z 16 320-339 185 185 185 316 316 316 F F F
208362024-02-04T14:09:25.130Z 17 340-359 187 187 187 319 319 319 F F F
208372024-02-04T14:09:25.130Z 18 360-379 178 178 178 306 306 306 F F F
208382024-02-04T14:09:25.130Z 19 380-399 193 193 193 325 325 325 F F F
208392024-02-04T14:09:25.130Z 20 400-419 179 179 179 307 307 307 F F F
208402024-02-04T14:09:25.130Z 21 420-439 193 193 193 325 325 325 F F F
208412024-02-04T14:09:25.130Z 22 440-459 193 193 193 325 325 325 F F F
208422024-02-04T14:09:25.130Z 23 460-479 192 192 192 324 324 324 F F F
208432024-02-04T14:09:25.130Z 24 480-499 192 192 192 324 324 324 F F F
208442024-02-04T14:09:25.130Z 25 500-519 190 190 190 322 322 322 F F F
208452024-02-04T14:09:25.131Z 26 520-539 191 191 191 323 323 323 F F F
208462024-02-04T14:09:25.131Z 27 540-559 191 191 191 323 323 323 F F F
208472024-02-04T14:09:25.131Z 28 560-579 193 193 193 325 325 325 F F F
208482024-02-04T14:09:25.131Z 29 580-599 192 192 192 324 324 324 F F F
208492024-02-04T14:09:25.131ZMax gen: 193, Max flush: 325
208502024-02-04T14:09:25.134Z
208512024-02-04T14:09:25.134ZBegin loop 97
208522024-02-04T14:09:25.134ZDownstairs to restart: 2
208532024-02-04T14:09:25.165Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.197251261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908}
208542024-02-04T14:09:25.165Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.197716392Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908}
208552024-02-04T14:09:25.165Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.197746518Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908}
208562024-02-04T14:09:25.165Z{"msg":"Crucible d907c90e-832d-4c26-b35f-eafe71435db6 has session id: 97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.197768251Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208572024-02-04T14:09:25.165Z{"msg":"Upstairs opts: Upstairs UUID: d907c90e-832d-4c26-b35f-eafe71435db6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.197802915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208582024-02-04T14:09:25.170Z{"msg":"Crucible stats registered with UUID: d907c90e-832d-4c26-b35f-eafe71435db6","v":0,"name":"crucible","level":30Crucible runtime is spawned
208592024-02-04T14:09:25.170Z,"time":"2024-02-04T14:09:24.197929018Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208602024-02-04T14:09:25.170Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.198255515Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908}
208612024-02-04T14:09:25.170Z{"msg":"d907c90e-832d-4c26-b35f-eafe71435db6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.19839813Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208622024-02-04T14:09:25.170Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.198525603Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208632024-02-04T14:09:25.170Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.198658099Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208642024-02-04T14:09:25.170Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.198701358Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208652024-02-04T14:09:25.170Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.198791281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208662024-02-04T14:09:25.170Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.198838698Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208672024-02-04T14:09:25.170Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.198984953Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208682024-02-04T14:09:25.170Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.199172403Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208692024-02-04T14:09:25.170Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.19925893Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208702024-02-04T14:09:25.170Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.199389365Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208712024-02-04T14:09:25.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.199478596Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208722024-02-04T14:09:25.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.199523477Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208732024-02-04T14:09:25.170Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.199564447Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208742024-02-04T14:09:25.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.200278228Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208752024-02-04T14:09:25.170Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.200346539Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208762024-02-04T14:09:25.170Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.200372625Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208772024-02-04T14:09:25.170Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.200418818Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208782024-02-04T14:09:25.175Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.201489299Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208792024-02-04T14:09:25.175Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.201599507Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208802024-02-04T14:09:25.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.202529036Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208812024-02-04T14:09:25.175Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.202622342Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208822024-02-04T14:09:25.175Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.202726646Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208832024-02-04T14:09:25.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.2028051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208842024-02-04T14:09:25.175Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.202854931Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208852024-02-04T14:09:25.175Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.202925524Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208862024-02-04T14:09:25.175Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.202976072Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208872024-02-04T14:09:25.175Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.203031424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208882024-02-04T14:09:25.176Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.203619165Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208892024-02-04T14:09:25.176Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204313442Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208902024-02-04T14:09:25.176Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204391809Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208912024-02-04T14:09:25.176Z{"msg":"[0]R flush_numbers[0..12]: [301, 324, 324, 325, 325, 324, 324, 320, 315, 319, 319, 324]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204458531Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208922024-02-04T14:09:25.176Z{"msg":"[0]R generation[0..12]: [176, 192, 192, 193, 193, 192, 192, 188, 184, 187, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204488424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208932024-02-04T14:09:25.176Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204531717Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208942024-02-04T14:09:25.176Z{"msg":"[1]R flush_numbers[0..12]: [301, 324, 324, 325, 325, 324, 324, 320, 315, 319, 319, 324]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
208952024-02-04T14:09:25.176Z,"time":"2024-02-04T14:09:24.204598316Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208962024-02-04T14:09:25.176Z{"msg":"[1]R generation[0..12]: [176, 192, 192, 193, 193, 192, 192, 188, 184, 187, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204672846Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208972024-02-04T14:09:25.176Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204742609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208982024-02-04T14:09:25.176Z{"msg":"[2]R flush_numbers[0..12]: [301, 324, 324, 325, 325, 324, 324, 320, 315, 319, 319, 324]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204768896Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
208992024-02-04T14:09:25.176Z{"msg":"[2]R generation[0..12]: [176, 192, 192, 193, 193, 192, 192, 188, 184, 187, 187, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204838214Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209002024-02-04T14:09:25.176Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204907878Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209012024-02-04T14:09:25.176Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204935259Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209022024-02-04T14:09:25.176Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.204991423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209032024-02-04T14:09:25.176Z{"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.205033162Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209042024-02-04T14:09:25.176Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.205102995Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209052024-02-04T14:09:25.176Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.205162815Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209062024-02-04T14:09:25.176Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.205185942Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209072024-02-04T14:09:25.176Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.2052137Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209082024-02-04T14:09:25.176Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.205281804Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209092024-02-04T14:09:25.176Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.205348474Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209102024-02-04T14:09:25.176Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.205388967Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209112024-02-04T14:09:25.176ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
209122024-02-04T14:09:25.177Z{"msg":"d907c90e-832d-4c26-b35f-eafe71435db6 is now active with session: 97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.2054724Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209132024-02-04T14:09:25.177Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.205541111Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908,"session_id":"97a1a7c7-2f73-4cd8-8db0-fee1e2a20f94"}
209142024-02-04T14:09:25.177Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:24.20561023Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908}
209152024-02-04T14:09:25.177ZLoading write count information from file "/tmp/test_repair_verify.data"
209162024-02-04T14:09:25.177ZRun Repair workload
209172024-02-04T14:09:25.177Z01/30 Flush
209182024-02-04T14:09:25.183Z02/30 Write block 329 len 40960 data: 9 10 11 11 12 14 12 10 10 11
209192024-02-04T14:09:25.183Z03/30 Read block 151 len 16384
209202024-02-04T14:09:25.183Z04/30 Read block 153 len 4096
209212024-02-04T14:09:25.186Z05/30 Read block 364 len 20480
209222024-02-04T14:09:25.186Z06/30 Read block 342 len 24576
209232024-02-04T14:09:25.186Z07/30 Read block 500 len 40960
209242024-02-04T14:09:25.186Z08/30 Flush
209252024-02-04T14:09:25.189Z09/30 Flush
209262024-02-04T14:09:25.192Z10/30 Write block 36 len 16384 data: 21 20 18 17
209272024-02-04T14:09:25.192Z11/30 Flush
209282024-02-04T14:09:25.195Z12/30 Write block 207 len 32768 data: 17 15 14 12 15 15 11 12
209292024-02-04T14:09:25.198Z13/30 Write block 561 len 20480 data: 14 12 15 18 19
209302024-02-04T14:09:25.198Z14/30 Write block 11 len 16384 data: 8 5 5 6
209312024-02-04T14:09:25.198Z15/30 Read block 18 len 36864
209322024-02-04T14:09:25.201Z16/30 Flush
209332024-02-04T14:09:25.204Z17/30 Write block 7 len 20480 data: 13 12 13 9 9
209342024-02-04T14:09:25.204Z18/30 Read block 502 len 28672
209352024-02-04T14:09:25.207Z19/30 Write block 434 len 28672 data: 19 19 18 16 12 12 16
209362024-02-04T14:09:25.207Z20/30 Read block 122 len 40960
209372024-02-04T14:09:25.210Z21/30 Read block 244 len 8192
209382024-02-04T14:09:25.210Z22/30 Read block 388 len 24576
209392024-02-04T14:09:25.213Z23/30 Read block 287 len 12288
209402024-02-04T14:09:25.213Z24/30 Write block 391 len 16384 data: 14 15 17 16
209412024-02-04T14:09:25.216Z25/30 Read block 258 len 8192
209422024-02-04T14:09:25.216Z26/30 Read block 498 len 40960
209432024-02-04T14:09:25.220Z27/30 Read block 406 len 40960
209442024-02-04T14:09:25.220Z28/30 Write block 172 len 20480 data: 17 15 14 14 14
209452024-02-04T14:09:25.226Z29/30 Write block 341 len 24576 data: 15 13 13 13 15 14
209462024-02-04T14:09:25.226Z30/30 Write block 187 len 12288 data: 19 16 14
209472024-02-04T14:09:25.226Z----------------------------------------------------------------
209482024-02-04T14:09:25.226Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:21
209492024-02-04T14:09:25.226ZStates: Active Active Active
209502024-02-04T14:09:25.226ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209512024-02-04T14:09:25.226Z 10 Acked 1009 Write 4 Done Done Done false
209522024-02-04T14:09:25.226Z 11 Acked 1010 Flush 0 Done Done Sent false
209532024-02-04T14:09:25.227Z 12 Acked 1011 Write 8 Done Done Sent false
209542024-02-04T14:09:25.227Z 13 Acked 1012 Write 5 Done Done Sent false
209552024-02-04T14:09:25.227Z 14 Acked 1013 Write 4 Done Done Sent false
209562024-02-04T14:09:25.227Z 15 Acked 1014 Read 9 Done Done Sent false
209572024-02-04T14:09:25.227Z 16 Acked 1015 Flush 0 Done Done Sent false
209582024-02-04T14:09:25.227Z 17 Acked 1016 Write 5 Done Done Sent false
209592024-02-04T14:09:25.227Z 18 Acked 1017 Read 7 Done Done Sent false
209602024-02-04T14:09:25.227Z 19 Acked 1018 Write 7 Done Done Sent false
209612024-02-04T14:09:25.227Z 20 Acked 1019 Read 10 Done Done Sent false
209622024-02-04T14:09:25.227Z 21 Acked 1020 Read 2 Done Done Sent false
209632024-02-04T14:09:25.227Z 22 Acked 1021 Read 6 Done Done Sent false
209642024-02-04T14:09:25.227Z 23 Acked 1022 Read 3 Done Done Sent false
209652024-02-04T14:09:25.227Z 24 Acked 1023 Write 4 Done Done Sent false
209662024-02-04T14:09:25.227Z 25 Acked 1024 Read 2 Done Done Sent false
209672024-02-04T14:09:25.227Z 26 Acked 1025 Read 10 Done Done Sent false
209682024-02-04T14:09:25.227Z 27 Acked 1026 Read 10 Done Done Sent false
209692024-02-04T14:09:25.227Z 28 Acked 1027 Write 5 Done Done Sent false
209702024-02-04T14:09:25.227Z 29 Acked 1028 Write 6 Sent Sent Sent false
209712024-02-04T14:09:25.227Z 30 Acked 1029 Write 3 Sent Sent Sent false
209722024-02-04T14:09:25.227Z STATES DS:0 DS:1 DS:2 TOTAL
209732024-02-04T14:09:25.227Z New 0 0 0 0
209742024-02-04T14:09:25.227Z Sent 2 2 20 24
209752024-02-04T14:09:25.227Z Done 19 19 1 39
209762024-02-04T14:09:25.227Z Skipped 0 0 0 0
209772024-02-04T14:09:25.227Z Error 0 0 0 0
209782024-02-04T14:09:25.228ZLast Flush: 1015 1015 1008
209792024-02-04T14:09:25.228ZDownstairs last five completed: 1008 1007 1006 1005 1004
209802024-02-04T14:09:25.228ZUpstairs last five completed: 30 29 28 27 26
209812024-02-04T14:09:25.228Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:09:24.254740468Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1908}
209822024-02-04T14:09:25.228ZWrote out file "/tmp/test_repair_verify.data"
209832024-02-04T14:09:25.231Z
209842024-02-04T14:09:25.236ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209852024-02-04T14:09:25.264ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209862024-02-04T14:09:25.264Z 0 000-019 194 194 176 330 330 301 T T F <---
209872024-02-04T14:09:25.265Z 1 020-039 194 194 192 329 329 324 F F T <---
209882024-02-04T14:09:25.265Z 2 040-059 192 192 192 324 324 324 F F F
209892024-02-04T14:09:25.265Z 3 060-079 193 193 193 325 325 325 F F F
209902024-02-04T14:09:25.265Z 4 080-099 193 193 193 325 325 325 F F F
209912024-02-04T14:09:25.265Z 5 100-119 192 192 192 324 324 324 F F F
209922024-02-04T14:09:25.265Z 6 120-139 192 192 192 324 324 324 F F F
209932024-02-04T14:09:25.265Z 7 140-159 188 188 188 320 320 320 F F F
209942024-02-04T14:09:25.265Z 8 160-179 184 184 184 315 315 315 T T F <---
209952024-02-04T14:09:25.268Z 9 180-199 187 187 187 319 319 319 T T F <---
209962024-02-04T14:09:25.268Z 10 200-219 194 194 187 330 330 319 F F F <---
209972024-02-04T14:09:25.268Z 11 220-239 192 192 192 324 324 324 F F F
209982024-02-04T14:09:25.268Z 12 240-259 190 190 190 322 322 322 F F F
209992024-02-04T14:09:25.268Z 13 260-279 188 188 188 320 320 320 F F F
210002024-02-04T14:09:25.268Z 14 280-299 188 188 188 320 320 320 F F F
210012024-02-04T14:09:25.268Z 15 300-319 192 192 192 324 324 324 F F F
210022024-02-04T14:09:25.268Z 16 320-339 194 194 194 327 327 327 F F F
210032024-02-04T14:09:25.268Z 17 340-359 187 187 187 319 319 319 T T F <---
210042024-02-04T14:09:25.268Z 18 360-379 178 178 178 306 306 306 F F F
210052024-02-04T14:09:25.268Z 19 380-399 193 193 193 325 325 325 T T F <---
210062024-02-04T14:09:25.268Z 20 400-419 179 179 179 307 307 307 F F F
210072024-02-04T14:09:25.268Z 21 420-439 193 193 193 325 325 325 T T F <---
210082024-02-04T14:09:25.269Z 22 440-459 193 193 193 325 325 325 T T F <---
210092024-02-04T14:09:25.269Z 23 460-479 192 192 192 324 324 324 F F F
210102024-02-04T14:09:25.269Z 24 480-499 192 192 192 324 324 324 F F F
210112024-02-04T14:09:25.269Z 25 500-519 190 190 190 322 322 322 F F F
210122024-02-04T14:09:25.269Z 26 520-539 191 191 191 323 323 323 F F F
210132024-02-04T14:09:25.269Z 27 540-559 191 191 191 323 323 323 F F F
210142024-02-04T14:09:25.269Z 28 560-579 194 194 193 330 330 325 F F F <---
210152024-02-04T14:09:25.269Z 29 580-599 192 192 192 324 324 324 F F F
210162024-02-04T14:09:25.269ZMax gen: 194, Max flush: 330
210172024-02-04T14:09:25.269ZError: Difference in extent metadata found!
210182024-02-04T14:09:25.272ZA Difference in extent metadata is expected here
210192024-02-04T14:09:25.272ZOn loop 97
210202024-02-04T14:09:25.272Z
210212024-02-04T14:09:25.275ZVerifying data now
210222024-02-04T14:09:30.211ZNo change after verify
210232024-02-04T14:09:30.211ZLoop: 97 Downstairs dump after verify (and repair):
210242024-02-04T14:09:30.234ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210252024-02-04T14:09:30.234Z 0 000-019 195 195 195 331 331 331 F F F
210262024-02-04T14:09:30.234Z 1 020-039 194 194 194 329 329 329 F F F
210272024-02-04T14:09:30.234Z 2 040-059 192 192 192 324 324 324 F F F
210282024-02-04T14:09:30.234Z 3 060-079 193 193 193 325 325 325 F F F
210292024-02-04T14:09:30.234Z 4 080-099 193 193 193 325 325 325 F F F
210302024-02-04T14:09:30.234Z 5 100-119 192 192 192 324 324 324 F F F
210312024-02-04T14:09:30.234Z 6 120-139 192 192 192 324 324 324 F F F
210322024-02-04T14:09:30.234Z 7 140-159 188 188 188 320 320 320 F F F
210332024-02-04T14:09:30.234Z 8 160-179 195 195 195 331 331 331 F F F
210342024-02-04T14:09:30.234Z 9 180-199 195 195 195 331 331 331 F F F
210352024-02-04T14:09:30.234Z 10 200-219 194 194 194 330 330 330 F F F
210362024-02-04T14:09:30.238Z 11 220-239 192 192 192 324 324 324 F F F
210372024-02-04T14:09:30.238Z 12 240-259 190 190 190 322 322 322 F F F
210382024-02-04T14:09:30.238Z 13 260-279 188 188 188 320 320 320 F F F
210392024-02-04T14:09:30.238Z 14 280-299 188 188 188 320 320 320 F F F
210402024-02-04T14:09:30.238Z 15 300-319 192 192 192 324 324 324 F F F
210412024-02-04T14:09:30.238Z 16 320-339 194 194 194 327 327 327 F F F
210422024-02-04T14:09:30.238Z 17 340-359 195 195 195 331 331 331 F F F
210432024-02-04T14:09:30.238Z 18 360-379 178 178 178 306 306 306 F F F
210442024-02-04T14:09:30.238Z 19 380-399 195 195 195 331 331 331 F F F
210452024-02-04T14:09:30.238Z 20 400-419 179 179 179 307 307 307 F F F
210462024-02-04T14:09:30.238Z 21 420-439 195 195 195 331 331 331 F F F
210472024-02-04T14:09:30.238Z 22 440-459 195 195 195 331 331 331 F F F
210482024-02-04T14:09:30.238Z 23 460-479 192 192 192 324 324 324 F F F
210492024-02-04T14:09:30.238Z 24 480-499 192 192 192 324 324 324 F F F
210502024-02-04T14:09:30.238Z 25 500-519 190 190 190 322 322 322 F F F
210512024-02-04T14:09:30.238Z 26 520-539 191 191 191 323 323 323 F F F
210522024-02-04T14:09:30.238Z 27 540-559 191 191 191 323 323 323 F F F
210532024-02-04T14:09:30.238Z 28 560-579 194 194 194 330 330 330 F F F
210542024-02-04T14:09:30.239Z 29 580-599 192 192 192 324 324 324 F F F
210552024-02-04T14:09:30.239ZMax gen: 195, Max flush: 331
210562024-02-04T14:09:30.242Z
210572024-02-04T14:09:30.242ZBegin loop 98
210582024-02-04T14:09:30.242ZDownstairs to restart: 2
210592024-02-04T14:09:30.269Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.300800252Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916}
210602024-02-04T14:09:30.273Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.301303794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916}
210612024-02-04T14:09:30.273Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.301389398Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916}
210622024-02-04T14:09:30.273Z{"msg":"Crucible 548698b1-6fb9-43ec-a7c7-6e1155535a21 has session id: 8496abdb-e80f-40e0-a7b6-868c9737cbd0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.301431402Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210632024-02-04T14:09:30.273Z{"msg":"Crucible runtime is spawned
210642024-02-04T14:09:30.273ZUpstairs opts: Upstairs UUID: 548698b1-6fb9-43ec-a7c7-6e1155535a21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.301615723Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210652024-02-04T14:09:30.273Z{"msg":"Crucible stats registered with UUID: 548698b1-6fb9-43ec-a7c7-6e1155535a21","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.301694502Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210662024-02-04T14:09:30.273Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.301735639Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916}
210672024-02-04T14:09:30.273Z{"msg":"548698b1-6fb9-43ec-a7c7-6e1155535a21 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.301932949Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210682024-02-04T14:09:30.273Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.301997671Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210692024-02-04T14:09:30.273Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302025516Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210702024-02-04T14:09:30.273Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302048674Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210712024-02-04T14:09:30.273Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302072196Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210722024-02-04T14:09:30.273Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302120261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210732024-02-04T14:09:30.274Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.30226556Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210742024-02-04T14:09:30.274Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302405523Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210752024-02-04T14:09:30.274Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.30254782Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210762024-02-04T14:09:30.274Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302621302Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210772024-02-04T14:09:30.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302665281Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"io task","client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210782024-02-04T14:09:30.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302736759Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"io task","client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210792024-02-04T14:09:30.274Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.302779338Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"io task","client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210802024-02-04T14:09:30.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.30404225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210812024-02-04T14:09:30.274Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.304149555Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210822024-02-04T14:09:30.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.304300794Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210832024-02-04T14:09:30.274Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.304349696Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210842024-02-04T14:09:30.279Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.304546915Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210852024-02-04T14:09:30.279Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.304692777Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210862024-02-04T14:09:30.279Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.305344731Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210872024-02-04T14:09:30.279Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.305545902Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210882024-02-04T14:09:30.279Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.305738867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210892024-02-04T14:09:30.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306409059Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210902024-02-04T14:09:30.279Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306471844Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210912024-02-04T14:09:30.279Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306497933Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210922024-02-04T14:09:30.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306565212Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210932024-02-04T14:09:30.279Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306589423Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210942024-02-04T14:09:30.279Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306611842Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210952024-02-04T14:09:30.279Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306727879Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210962024-02-04T14:09:30.279Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306764791Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210972024-02-04T14:09:30.280Z{"msg":"[0]R flush_numbers[0..12]: [331, 329, 324, 325, 325, 324, 324, 320, 331, 331, 330, 324]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306845253Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210982024-02-04T14:09:30.280Z{"msg":"[0]R generation[0..12]: [195, 194, 192, 193, 193, 192, 192, 188, 195, 195, 194, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306891772Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
210992024-02-04T14:09:30.280Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306917722Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211002024-02-04T14:09:30.280Z{"msg":"[1]R flush_numbers[0..12]: [331, 329, 324, 325, 325, 324, 324, 320, 331, 331, 330, 324]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306941679Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211012024-02-04T14:09:30.280Z{"msg":"[1]R generation[0..12]: [195, 194, 192, 193, 193, 192, 192, 188, 195, 195, 194, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.306965461Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211022024-02-04T14:09:30.280Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307007879Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211032024-02-04T14:09:30.280Z{"msg":"[2]R flush_numbers[0..12]: [331, 329, 324, 325, 325, 324, 324, 320, 331, 331, 330, 324]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307032889Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211042024-02-04T14:09:30.280Z{"msg":"[2]R generation[0..12]: [195, 194, 192, 193, 193, 192, 192, 188, 195, 195, 194, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.30705628Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211052024-02-04T14:09:30.280ZWait for a query_work_queue command to finish before sending IO
211062024-02-04T14:09:30.280Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307104175Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211072024-02-04T14:09:30.280Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307127843Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211082024-02-04T14:09:30.280Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307150243Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211092024-02-04T14:09:30.280Z{"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307192912Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211102024-02-04T14:09:30.280Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307215819Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"":"downstairs","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211112024-02-04T14:09:30.280Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307238277Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211122024-02-04T14:09:30.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307277379Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"0","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211132024-02-04T14:09:30.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307300815Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"1","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211142024-02-04T14:09:30.280Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307323176Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"client":"2","":"downstairs","session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211152024-02-04T14:09:30.280Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.3073456Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211162024-02-04T14:09:30.280Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.30738536Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211172024-02-04T14:09:30.280Z{"msg":"548698b1-6fb9-43ec-a7c7-6e1155535a21 is now active with session: 8496abdb-e80f-40e0-a7b6-868c9737cbd0","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307408849Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211182024-02-04T14:09:30.280Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307451138Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916,"session_id":"8496abdb-e80f-40e0-a7b6-868c9737cbd0"}
211192024-02-04T14:09:30.280Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:29.307473218Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916}
211202024-02-04T14:09:30.281ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211212024-02-04T14:09:30.281ZLoading write count information from file "/tmp/test_repair_verify.data"
211222024-02-04T14:09:30.281ZRun Repair workload
211232024-02-04T14:09:30.281Z01/30 Write block 213 len 16384 data: 12 13 9 11
211242024-02-04T14:09:30.284Z02/30 Flush
211252024-02-04T14:09:30.284Z03/30 Write block 423 len 40960 data: 14 10 12 11 13 14 13 16 17 16
211262024-02-04T14:09:30.284Z04/30 Write block 536 len 12288 data: 12 9 12
211272024-02-04T14:09:30.284Z05/30 Write block 252 len 28672 data: 16 13 16 16 15 12 14
211282024-02-04T14:09:30.291Z06/30 Read block 54 len 28672
211292024-02-04T14:09:30.291Z07/30 Read block 209 len 4096
211302024-02-04T14:09:30.294Z08/30 Write block 218 len 20480 data: 9 10 13 14 11
211312024-02-04T14:09:30.294Z09/30 Read block 285 len 20480
211322024-02-04T14:09:30.294Z10/30 Read block 226 len 16384
211332024-02-04T14:09:30.294Z11/30 Flush
211342024-02-04T14:09:30.297Z12/30 Write block 140 len 36864 data: 19 14 10 11 13 14 11 9 8
211352024-02-04T14:09:30.297Z13/30 Read block 232 len 4096
211362024-02-04T14:09:30.300Z14/30 Read block 25 len 20480
211372024-02-04T14:09:30.300Z15/30 Read block 536 len 12288
211382024-02-04T14:09:30.303Z16/30 Read block 494 len 32768
211392024-02-04T14:09:30.306Z17/30 Flush
211402024-02-04T14:09:30.306Z18/30 Write block 310 len 8192 data: 18 17
211412024-02-04T14:09:30.309Z19/30 Read block 582 len 32768
211422024-02-04T14:09:30.309Z20/30 Read block 233 len 40960
211432024-02-04T14:09:30.312Z21/30 Write block 534 len 32768 data: 10 10 13 10 13 13 15 16
211442024-02-04T14:09:30.312Z22/30 Read block 254 len 28672
211452024-02-04T14:09:30.315Z23/30 Flush
211462024-02-04T14:09:30.318Z24/30 Write block 44 len 16384 data: 15 16 16 13
211472024-02-04T14:09:30.321Z25/30 Write block 266 len 40960 data: 17 15 20 19 20 17 12 13 11 10
211482024-02-04T14:09:30.321Z26/30 Read block 122 len 28672
211492024-02-04T14:09:30.324Z27/30 Write block 7 len 24576 data: 14 13 14 10 10 6
211502024-02-04T14:09:30.324Z28/30 Read block 439 len 12288
211512024-02-04T14:09:30.327Z29/30 Read block 303 len 36864
211522024-02-04T14:09:30.327Z30/30 Read block 368 len 8192
211532024-02-04T14:09:30.330Z----------------------------------------------------------------
211542024-02-04T14:09:30.330Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:28
211552024-02-04T14:09:30.330ZStates: Active Active Active
211562024-02-04T14:09:30.330ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211572024-02-04T14:09:30.330Z 3 Acked 1002 Write 10 Done Done Sent false
211582024-02-04T14:09:30.331Z 4 Acked 1003 Write 3 Done Done Sent false
211592024-02-04T14:09:30.331Z 5 Acked 1004 Write 7 Done Done Sent false
211602024-02-04T14:09:30.331Z 6 Acked 1005 Read 7 Done Done Sent false
211612024-02-04T14:09:30.331Z 7 Acked 1006 Read 1 Done Done Sent false
211622024-02-04T14:09:30.331Z 8 Acked 1007 Write 5 Done Done Sent false
211632024-02-04T14:09:30.331Z 9 Acked 1008 Read 5 Done Done Sent false
211642024-02-04T14:09:30.331Z 10 Acked 1009 Read 4 Done Done Sent false
211652024-02-04T14:09:30.331Z 11 Acked 1010 Flush 0 Done Done Sent false
211662024-02-04T14:09:30.331Z 12 Acked 1011 Write 9 Done Done Sent false
211672024-02-04T14:09:30.331Z 13 Acked 1012 Read 1 Done Done Sent false
211682024-02-04T14:09:30.331Z 14 Acked 1013 Read 5 Done Done Sent false
211692024-02-04T14:09:30.331Z 15 Acked 1014 Read 3 Done Done Sent false
211702024-02-04T14:09:30.331Z 16 Acked 1015 Read 8 Done Done Sent false
211712024-02-04T14:09:30.331Z 17 Acked 1016 Flush 0 Done Done Sent false
211722024-02-04T14:09:30.331Z 18 Acked 1017 Write 2 Done Done Sent false
211732024-02-04T14:09:30.331Z 19 Acked 1018 Read 8 Done Done Sent false
211742024-02-04T14:09:30.331Z 20 Acked 1019 Read 10 Done Done Sent false
211752024-02-04T14:09:30.331Z 21 Acked 1020 Write 8 Done Done Sent false
211762024-02-04T14:09:30.331Z 22 Acked 1021 Read 7 Done Done Sent false
211772024-02-04T14:09:30.331Z 23 Acked 1022 Flush 0 Done Done Sent false
211782024-02-04T14:09:30.331Z 24 Acked 1023 Write 4 Done Done Sent false
211792024-02-04T14:09:30.331Z 25 Acked 1024 Write 10 Done Done Sent false
211802024-02-04T14:09:30.331Z 26 Acked 1025 Read 7 Done Done Sent false
211812024-02-04T14:09:30.331Z 27 Acked 1026 Write 6 Done Done Sent false
211822024-02-04T14:09:30.331Z 28 Acked 1027 Read 3 Done Done Sent false
211832024-02-04T14:09:30.332Z 29 Acked 1028 Read 9 Done Done Sent false
211842024-02-04T14:09:30.332Z 30 Acked 1029 Read 2 Done Done Sent false
211852024-02-04T14:09:30.332Z STATES DS:0 DS:1 DS:2 TOTAL
211862024-02-04T14:09:30.332Z New 0 0 0 0
211872024-02-04T14:09:30.332Z Sent 0 0 28 28
211882024-02-04T14:09:30.332Z Done 28 28 0 56
211892024-02-04T14:09:30.332Z Skipped 0 0 0 0
211902024-02-04T14:09:30.332Z Error 0 0 0 0
211912024-02-04T14:09:30.332ZLast Flush: 1022 1022 1001
211922024-02-04T14:09:30.332ZDownstairs last five completed: 1001 1000
211932024-02-04T14:09:30.332ZUpstairs last five completed: 30 29 28 27 26
211942024-02-04T14:09:30.332Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:09:29.361260759Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1916}
211952024-02-04T14:09:30.332ZWrote out file "/tmp/test_repair_verify.data"
211962024-02-04T14:09:30.336Z
211972024-02-04T14:09:30.343ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211982024-02-04T14:09:30.370ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211992024-02-04T14:09:30.370Z 0 000-019 195 195 195 331 331 331 T T F <---
212002024-02-04T14:09:30.370Z 1 020-039 194 194 194 329 329 329 F F F
212012024-02-04T14:09:30.370Z 2 040-059 192 192 192 324 324 324 T T F <---
212022024-02-04T14:09:30.370Z 3 060-079 193 193 193 325 325 325 F F F
212032024-02-04T14:09:30.370Z 4 080-099 193 193 193 325 325 325 F F F
212042024-02-04T14:09:30.371Z 5 100-119 192 192 192 324 324 324 F F F
212052024-02-04T14:09:30.371Z 6 120-139 192 192 192 324 324 324 F F F
212062024-02-04T14:09:30.371Z 7 140-159 196 196 188 334 334 320 F F F <---
212072024-02-04T14:09:30.371Z 8 160-179 195 195 195 331 331 331 F F F
212082024-02-04T14:09:30.371Z 9 180-199 195 195 195 331 331 331 F F F
212092024-02-04T14:09:30.371Z 10 200-219 196 196 196 333 333 332 F F F <---
212102024-02-04T14:09:30.371Z 11 220-239 196 196 192 333 333 324 F F F <---
212112024-02-04T14:09:30.371Z 12 240-259 196 196 190 333 333 322 F F F <---
212122024-02-04T14:09:30.374Z 13 260-279 188 188 188 320 320 320 T T F <---
212132024-02-04T14:09:30.374Z 14 280-299 188 188 188 320 320 320 F F F
212142024-02-04T14:09:30.374Z 15 300-319 196 196 192 335 335 324 F F F <---
212152024-02-04T14:09:30.374Z 16 320-339 194 194 194 327 327 327 F F F
212162024-02-04T14:09:30.374Z 17 340-359 195 195 195 331 331 331 F F F
212172024-02-04T14:09:30.375Z 18 360-379 178 178 178 306 306 306 F F F
212182024-02-04T14:09:30.375Z 19 380-399 195 195 195 331 331 331 F F F
212192024-02-04T14:09:30.375Z 20 400-419 179 179 179 307 307 307 F F F
212202024-02-04T14:09:30.375Z 21 420-439 196 196 195 333 333 331 F F F <---
212212024-02-04T14:09:30.375Z 22 440-459 195 195 195 331 331 331 F F F
212222024-02-04T14:09:30.375Z 23 460-479 192 192 192 324 324 324 F F F
212232024-02-04T14:09:30.375Z 24 480-499 192 192 192 324 324 324 F F F
212242024-02-04T14:09:30.375Z 25 500-519 190 190 190 322 322 322 F F F
212252024-02-04T14:09:30.375Z 26 520-539 196 196 191 335 335 323 F F F <---
212262024-02-04T14:09:30.375Z 27 540-559 196 196 191 335 335 323 F F F <---
212272024-02-04T14:09:30.375Z 28 560-579 194 194 194 330 330 330 F F F
212282024-02-04T14:09:30.375Z 29 580-599 192 192 192 324 324 324 F F F
212292024-02-04T14:09:30.375ZMax gen: 196, Max flush: 335
212302024-02-04T14:09:30.375ZError: Difference in extent metadata found!
212312024-02-04T14:09:30.375ZA Difference in extent metadata is expected here
212322024-02-04T14:09:30.375ZOn loop 98
212332024-02-04T14:09:30.375Z
212342024-02-04T14:09:30.379ZVerifying data now
212352024-02-04T14:09:35.367ZNo change after verify
212362024-02-04T14:09:35.367ZLoop: 98 Downstairs dump after verify (and repair):
212372024-02-04T14:09:35.391ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212382024-02-04T14:09:35.391Z 0 000-019 197 197 197 336 336 336 F F F
212392024-02-04T14:09:35.391Z 1 020-039 194 194 194 329 329 329 F F F
212402024-02-04T14:09:35.391Z 2 040-059 197 197 197 336 336 336 F F F
212412024-02-04T14:09:35.391Z 3 060-079 193 193 193 325 325 325 F F F
212422024-02-04T14:09:35.391Z 4 080-099 193 193 193 325 325 325 F F F
212432024-02-04T14:09:35.391Z 5 100-119 192 192 192 324 324 324 F F F
212442024-02-04T14:09:35.391Z 6 120-139 192 192 192 324 324 324 F F F
212452024-02-04T14:09:35.397Z 7 140-159 196 196 196 334 334 334 F F F
212462024-02-04T14:09:35.397Z 8 160-179 195 195 195 331 331 331 F F F
212472024-02-04T14:09:35.397Z 9 180-199 195 195 195 331 331 331 F F F
212482024-02-04T14:09:35.397Z 10 200-219 196 196 196 333 333 333 F F F
212492024-02-04T14:09:35.397Z 11 220-239 196 196 196 333 333 333 F F F
212502024-02-04T14:09:35.397Z 12 240-259 196 196 196 333 333 333 F F F
212512024-02-04T14:09:35.397Z 13 260-279 197 197 197 336 336 336 F F F
212522024-02-04T14:09:35.398Z 14 280-299 188 188 188 320 320 320 F F F
212532024-02-04T14:09:35.398Z 15 300-319 196 196 196 335 335 335 F F F
212542024-02-04T14:09:35.398Z 16 320-339 194 194 194 327 327 327 F F F
212552024-02-04T14:09:35.398Z 17 340-359 195 195 195 331 331 331 F F F
212562024-02-04T14:09:35.398Z 18 360-379 178 178 178 306 306 306 F F F
212572024-02-04T14:09:35.398Z 19 380-399 195 195 195 331 331 331 F F F
212582024-02-04T14:09:35.398Z 20 400-419 179 179 179 307 307 307 F F F
212592024-02-04T14:09:35.398Z 21 420-439 196 196 196 333 333 333 F F F
212602024-02-04T14:09:35.398Z 22 440-459 195 195 195 331 331 331 F F F
212612024-02-04T14:09:35.398Z 23 460-479 192 192 192 324 324 324 F F F
212622024-02-04T14:09:35.398Z 24 480-499 192 192 192 324 324 324 F F F
212632024-02-04T14:09:35.398Z 25 500-519 190 190 190 322 322 322 F F F
212642024-02-04T14:09:35.398Z 26 520-539 196 196 196 335 335 335 F F F
212652024-02-04T14:09:35.398Z 27 540-559 196 196 196 335 335 335 F F F
212662024-02-04T14:09:35.398Z 28 560-579 194 194 194 330 330 330 F F F
212672024-02-04T14:09:35.398Z 29 580-599 192 192 192 324 324 324 F F F
212682024-02-04T14:09:35.398ZMax gen: 197, Max flush: 336
212692024-02-04T14:09:35.401Z
212702024-02-04T14:09:35.402ZBegin loop 99
212712024-02-04T14:09:35.402ZDownstairs to restart: 1
212722024-02-04T14:09:35.429Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.460767604Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924}
212732024-02-04T14:09:35.429Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"93b33136315e8cafbb8542022f326676bfda840d\",\n git_commit_timestamp: \"2024-02-04T04:19:59.000000000Z\",\n git_branch: \"renovate/reedline-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.461224225Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924}
212742024-02-04T14:09:35.429Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.46126079Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924}
212752024-02-04T14:09:35.434Z{"msg":"Crucible 272ef24c-6fff-49c8-b7dc-38ada969e213 has session id: e77d3364-450d-4aad-9f78-312bd732c9ce","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.461347209Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212762024-02-04T14:09:35.434Z{"msg":"Upstairs opts: Upstairs UUID: 272ef24c-6fff-49c8-b7dc-38ada969e213, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.461405268Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212772024-02-04T14:09:35.434Z{Crucible runtime is spawned
212782024-02-04T14:09:35.434Z"msg":"Crucible stats registered with UUID: 272ef24c-6fff-49c8-b7dc-38ada969e213","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.461637002Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212792024-02-04T14:09:35.434Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.461858072Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924}
212802024-02-04T14:09:35.434Z{"msg":"272ef24c-6fff-49c8-b7dc-38ada969e213 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462012201Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212812024-02-04T14:09:35.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462141532Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212822024-02-04T14:09:35.434Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462189587Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212832024-02-04T14:09:35.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462339485Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212842024-02-04T14:09:35.434Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462555213Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212852024-02-04T14:09:35.434Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462700642Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212862024-02-04T14:09:35.434Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462776672Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212872024-02-04T14:09:35.434Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462820443Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212882024-02-04T14:09:35.434Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462860488Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212892024-02-04T14:09:35.434Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.462930123Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212902024-02-04T14:09:35.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.463000051Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"io task","client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212912024-02-04T14:09:35.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.463043892Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"io task","client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212922024-02-04T14:09:35.434Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.463085143Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"io task","client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212932024-02-04T14:09:35.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.463887238Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212942024-02-04T14:09:35.435Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.464048349Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212952024-02-04T14:09:35.435Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.464125304Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212962024-02-04T14:09:35.435Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.464172433Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212972024-02-04T14:09:35.435Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.464975867Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212982024-02-04T14:09:35.435Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.465130104Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
212992024-02-04T14:09:35.439Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.465661018Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213002024-02-04T14:09:35.439Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.465790942Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213012024-02-04T14:09:35.439Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.465988562Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213022024-02-04T14:09:35.439Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.466057399Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213032024-02-04T14:09:35.440Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.466084586Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213042024-02-04T14:09:35.440Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.466105979Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213052024-02-04T14:09:35.440Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.466128837Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213062024-02-04T14:09:35.440Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.466151255Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213072024-02-04T14:09:35.440Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.466528991Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213082024-02-04T14:09:35.440Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.46722062Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213092024-02-04T14:09:35.440Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467320088Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213102024-02-04T14:09:35.440Z{"msg":"[0]R flush_numbers[0..12]: [336, 329, 336, 325, 325, 324, 324, 334, 331, 331, 333, 333]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467347278Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213112024-02-04T14:09:35.440Z{"msg":"[0]R generation[0..12]: [197, 194, 197, 193, 193, 192, 192, 196, 195, 195, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467371525Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213122024-02-04T14:09:35.440Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467395785Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213132024-02-04T14:09:35.440Z{"msg":"[1]R flush_numbers[0..12]: [336, 329, 336, 325, 325, 324, 324, 334, 331, 331, 333, 333]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467439263Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213142024-02-04T14:09:35.440Z{"msg":"[1]R generation[0..12]: [197, 194, 197, 193, 193, 192, 192, 196, 195, 195, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467464418Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213152024-02-04T14:09:35.440Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.46750755Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213162024-02-04T14:09:35.440Z{"msg":"[2]R flush_numbers[0..12]: [336, 329, 336, 325, 325, 324, 324, 334, 331, 331, 333, 333]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467532851Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213172024-02-04T14:09:35.440Z{"msg":"[2]R generation[0..12]: [197, 194, 197, 193, 193, 192, 192, 196, 195, 195, 196, 196]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467574757Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO
213182024-02-04T14:09:35.440Z","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213192024-02-04T14:09:35.440Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467603902Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213202024-02-04T14:09:35.440Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467644782Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213212024-02-04T14:09:35.440Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467668609Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213222024-02-04T14:09:35.440Z{"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467709145Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213232024-02-04T14:09:35.440Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467799257Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"":"downstairs","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213242024-02-04T14:09:35.440Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467823283Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213252024-02-04T14:09:35.440Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467869424Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"0","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213262024-02-04T14:09:35.441Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467892971Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"1","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213272024-02-04T14:09:35.441Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467946624Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"client":"2","":"downstairs","session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213282024-02-04T14:09:35.441Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467970364Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213292024-02-04T14:09:35.441Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.467991588Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213302024-02-04T14:09:35.441Z{"msg":"272ef24c-6fff-49c8-b7dc-38ada969e213 is now active with session: e77d3364-450d-4aad-9f78-312bd732c9ce","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.46803261Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213312024-02-04T14:09:35.441Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.468053914Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924,"session_id":"e77d3364-450d-4aad-9f78-312bd732c9ce"}
213322024-02-04T14:09:35.441Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-04T14:09:34.468093311Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924}
213332024-02-04T14:09:35.441ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213342024-02-04T14:09:35.441ZLoading write count information from file "/tmp/test_repair_verify.data"
213352024-02-04T14:09:35.441ZRun Repair workload
213362024-02-04T14:09:35.441Z01/30 Flush
213372024-02-04T14:09:35.441Z02/30 Flush
213382024-02-04T14:09:35.444Z03/30 Write block 280 len 32768 data: 11 10 10 14 19 17 14 16
213392024-02-04T14:09:35.445Z04/30 Read block 104 len 8192
213402024-02-04T14:09:35.445Z05/30 Write block 428 len 8192 data: 15 14
213412024-02-04T14:09:35.445Z06/30 Read block 107 len 40960
213422024-02-04T14:09:35.447Z07/30 Write block 89 len 4096 data: 8
213432024-02-04T14:09:35.447Z08/30 Read block 576 len 40960
213442024-02-04T14:09:35.451Z09/30 Write block 215 len 4096 data: 10
213452024-02-04T14:09:35.451Z10/30 Read block 426 len 20480
213462024-02-04T14:09:35.451Z11/30 Read block 423 len 24576
213472024-02-04T14:09:35.454Z12/30 Write block 222 len 12288 data: 12 11 10
213482024-02-04T14:09:35.454Z13/30 Write block 500 len 20480 data: 20 19 15 15 16
213492024-02-04T14:09:35.454Z14/30 Read block 118 len 32768
213502024-02-04T14:09:35.456Z15/30 Write block 206 len 16384 data: 18 18 16 15
213512024-02-04T14:09:35.456Z16/30 Read block 311 len 28672
213522024-02-04T14:09:35.459Z17/30 Read block 233 len 32768
213532024-02-04T14:09:35.462Z18/30 Read block 61 len 32768
213542024-02-04T14:09:35.462Z19/30 Write block 381 len 8192 data: 10 8
213552024-02-04T14:09:35.465Z20/30 Write block 153 len 16384 data: 13 12 10 9
213562024-02-04T14:09:35.465Z21/30 Flush
213572024-02-04T14:09:35.468Z22/30 Write block 490 len 24576 data: 13 14 17 18 15 16
213582024-02-04T14:09:35.471Z23/30 Flush
213592024-02-04T14:09:35.474Z24/30 Flush
213602024-02-04T14:09:35.474Z25/30 Write block 173 len 8192 data: 16 15
213612024-02-04T14:09:35.474Z26/30 Write block 356 len 12288 data: 18 12 12
213622024-02-04T14:09:35.476Z27/30 Write block 192 len 24576 data: 12 14 13 12 11 14
213632024-02-04T14:09:35.477Z28/30 Read block 294 len 28672
213642024-02-04T14:09:35.480Z29/30 Write block 397 len 16384 data: 16 14 12 15
213652024-02-04T14:09:35.480Z30/30 Read block 449 len 16384
213662024-02-04T14:09:35.483Z----------------------------------------------------------------
213672024-02-04T14:09:35.483Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:29
213682024-02-04T14:09:35.483ZStates: Active Active Active
213692024-02-04T14:09:35.483ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213702024-02-04T14:09:35.483Z 2 Acked 1001 Flush 0 Done Sent Done false
213712024-02-04T14:09:35.483Z 3 Acked 1002 Write 8 Done Sent Done false
213722024-02-04T14:09:35.483Z 4 Acked 1003 Read 2 Done Sent Done false
213732024-02-04T14:09:35.483Z 5 Acked 1004 Write 2 Done Sent Done false
213742024-02-04T14:09:35.483Z 6 Acked 1005 Read 10 Done Sent Done false
213752024-02-04T14:09:35.483Z 7 Acked 1006 Write 1 Done Sent Done false
213762024-02-04T14:09:35.483Z 8 Acked 1007 Read 10 Done Sent Done false
213772024-02-04T14:09:35.484Z 9 Acked 1008 Write 1 Done Sent Done false
213782024-02-04T14:09:35.484Z 10 Acked 1009 Read 5 Done Sent Done false
213792024-02-04T14:09:35.484Z 11 Acked 1010 Read 6 Done Sent Done false
213802024-02-04T14:09:35.484Z 12 Acked 1011 Write 3 Done Sent Done false
213812024-02-04T14:09:35.484Z 13 Acked 1012 Write 5 Done Sent Done false
213822024-02-04T14:09:35.484Z 14 Acked 1013 Read 8 Done Sent Done false
213832024-02-04T14:09:35.484Z 15 Acked 1014 Write 4 Done Sent Done false
213842024-02-04T14:09:35.484Z 16 Acked 1015 Read 7 Done Sent Done false
213852024-02-04T14:09:35.484Z 17 Acked 1016 Read 8 Done Sent Done false
213862024-02-04T14:09:35.484Z 18 Acked 1017 Read 8 Done Sent Done false
213872024-02-04T14:09:35.484Z 19 Acked 1018 Write 2 Done Sent Done false
213882024-02-04T14:09:35.484Z 20 Acked 1019 Write 4 Done Sent Done false
213892024-02-04T14:09:35.484Z 21 Acked 1020 Flush 0 Done Sent Done false
213902024-02-04T14:09:35.484Z 22 Acked 1021 Write 6 Done Sent Done false
213912024-02-04T14:09:35.484Z 23 Acked 1022 Flush 0 Done Sent Done false
213922024-02-04T14:09:35.484Z 24 Acked 1023 Flush 0 Done Sent Done false
213932024-02-04T14:09:35.484Z 25 Acked 1024 Write 2 Done Sent Done false
213942024-02-04T14:09:35.484Z 26 Acked 1025 Write 3 Done Sent Done false
213952024-02-04T14:09:35.484Z 27 Acked 1026 Write 6 Done Sent Done false
213962024-02-04T14:09:35.484Z 28 Acked 1027 Read 7 Done Sent Done false
213972024-02-04T14:09:35.484Z 29 Acked 1028 Write 4 Done Sent Done false
213982024-02-04T14:09:35.484Z 30 Acked 1029 Read 4 Done Sent Sent false
213992024-02-04T14:09:35.484Z STATES DS:0 DS:1 DS:2 TOTAL
214002024-02-04T14:09:35.484Z New 0 0 0 0
214012024-02-04T14:09:35.485Z Sent 0 29 1 30
214022024-02-04T14:09:35.485Z Done 29 0 28 57
214032024-02-04T14:09:35.485Z Skipped 0 0 0 0
214042024-02-04T14:09:35.485Z Error 0 0 0 0
214052024-02-04T14:09:35.485ZLast Flush: 1023 1000 1023
214062024-02-04T14:09:35.485ZDownstairs last five completed: 1000
214072024-02-04T14:09:35.485ZUpstairs last five completed: 30 29 28 27 26
214082024-02-04T14:09:35.485Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-04T14:09:34.513251419Z","hostname":"ip-10-150-1-226.us-west-2.compute.internal","pid":1924}
214092024-02-04T14:09:35.485ZWrote out file "/tmp/test_repair_verify.data"
214102024-02-04T14:09:35.491Z
214112024-02-04T14:09:35.495ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
214122024-02-04T14:09:35.521ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214132024-02-04T14:09:35.522Z 0 000-019 197 197 197 336 336 336 F F F
214142024-02-04T14:09:35.522Z 1 020-039 194 194 194 329 329 329 F F F
214152024-02-04T14:09:35.522Z 2 040-059 197 197 197 336 336 336 F F F
214162024-02-04T14:09:35.522Z 3 060-079 193 193 193 325 325 325 F F F
214172024-02-04T14:09:35.522Z 4 080-099 198 193 198 339 325 339 F F F <---
214182024-02-04T14:09:35.522Z 5 100-119 192 192 192 324 324 324 F F F
214192024-02-04T14:09:35.522Z 6 120-139 192 192 192 324 324 324 F F F
214202024-02-04T14:09:35.522Z 7 140-159 198 196 198 339 334 339 F F F <---
214212024-02-04T14:09:35.522Z 8 160-179 195 195 195 331 331 331 T F T <---
214222024-02-04T14:09:35.522Z 9 180-199 195 195 195 331 331 331 T F T <---
214232024-02-04T14:09:35.522Z 10 200-219 198 196 198 339 333 339 F F F <---
214242024-02-04T14:09:35.522Z 11 220-239 198 196 198 339 333 339 F F F <---
214252024-02-04T14:09:35.531Z 12 240-259 196 196 196 333 333 333 F F F
214262024-02-04T14:09:35.531Z 13 260-279 197 197 197 336 336 336 F F F
214272024-02-04T14:09:35.531Z 14 280-299 198 188 198 339 320 339 F F F <---
214282024-02-04T14:09:35.532Z 15 300-319 196 196 196 335 335 335 F F F
214292024-02-04T14:09:35.532Z 16 320-339 194 194 194 327 327 327 F F F
214302024-02-04T14:09:35.532Z 17 340-359 195 195 195 331 331 331 T F T <---
214312024-02-04T14:09:35.532Z 18 360-379 178 178 178 306 306 306 F F F
214322024-02-04T14:09:35.532Z 19 380-399 198 195 198 339 331 339 T F T <---
214332024-02-04T14:09:35.532Z 20 400-419 179 179 179 307 307 307 T F T <---
214342024-02-04T14:09:35.532Z 21 420-439 198 196 198 339 333 339 F F F <---
214352024-02-04T14:09:35.532Z 22 440-459 195 195 195 331 331 331 F F F
214362024-02-04T14:09:35.532Z 23 460-479 192 192 192 324 324 324 F F F
214372024-02-04T14:09:35.532Z 24 480-499 198 192 198 340 324 340 F F F <---
214382024-02-04T14:09:35.532Z 25 500-519 198 190 198 339 322 339 F F F <---
214392024-02-04T14:09:35.532Z 26 520-539 196 196 196 335 335 335 F F F
214402024-02-04T14:09:35.532Z 27 540-559 196 196 196 335 335 335 F F F
214412024-02-04T14:09:35.532Z 28 560-579 194 194 194 330 330 330 F F F
214422024-02-04T14:09:35.532Z 29 580-599 192 192 192 324 324 324 F F F
214432024-02-04T14:09:35.532ZMax gen: 198, Max flush: 340
214442024-02-04T14:09:35.532ZError: Difference in extent metadata found!
214452024-02-04T14:09:35.535ZA Difference in extent metadata is expected here
214462024-02-04T14:09:35.535ZOn loop 99
214472024-02-04T14:09:35.535Z
214482024-02-04T14:09:35.535ZVerifying data now
214492024-02-04T14:09:40.682ZNo change after verify
214502024-02-04T14:09:40.682ZLoop: 99 Downstairs dump after verify (and repair):
214512024-02-04T14:09:40.709ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214522024-02-04T14:09:40.709Z 0 000-019 197 197 197 336 336 336 F F F
214532024-02-04T14:09:40.709Z 1 020-039 194 194 194 329 329 329 F F F
214542024-02-04T14:09:40.709Z 2 040-059 197 197 197 336 336 336 F F F
214552024-02-04T14:09:40.709Z 3 060-079 193 193 193 325 325 325 F F F
214562024-02-04T14:09:40.709Z 4 080-099 198 198 198 339 339 339 F F F
214572024-02-04T14:09:40.709Z 5 100-119 192 192 192 324 324 324 F F F
214582024-02-04T14:09:40.709Z 6 120-139 192 192 192 324 324 324 F F F
214592024-02-04T14:09:40.709Z 7 140-159 198 198 198 339 339 339 F F F
214602024-02-04T14:09:40.709Z 8 160-179 199 199 199 341 341 341 F F F
214612024-02-04T14:09:40.709Z 9 180-199 199 199 199 341 341 341 F F F
214622024-02-04T14:09:40.710Z 10 200-219 198 198 198 339 339 339 F F F
214632024-02-04T14:09:40.710Z 11 220-239 198 198 198 339 339 339 F F F
214642024-02-04T14:09:40.710Z 12 240-259 196 196 196 333 333 333 F F F
214652024-02-04T14:09:40.710Z 13 260-279 197 197 197 336 336 336 F F F
214662024-02-04T14:09:40.713Z 14 280-299 198 198 198 339 339 339 F F F
214672024-02-04T14:09:40.713Z 15 300-319 196 196 196 335 335 335 F F F
214682024-02-04T14:09:40.714Z 16 320-339 194 194 194 327 327 327 F F F
214692024-02-04T14:09:40.714Z 17 340-359 199 199 199 341 341 341 F F F
214702024-02-04T14:09:40.714Z 18 360-379 178 178 178 306 306 306 F F F
214712024-02-04T14:09:40.714Z 19 380-399 199 199 199 341 341 341 F F F
214722024-02-04T14:09:40.714Z 20 400-419 199 199 199 341 341 341 F F F
214732024-02-04T14:09:40.714Z 21 420-439 198 198 198 339 339 339 F F F
214742024-02-04T14:09:40.714Z 22 440-459 195 195 195 331 331 331 F F F
214752024-02-04T14:09:40.714Z 23 460-479 192 192 192 324 324 324 F F F
214762024-02-04T14:09:40.714Z 24 480-499 198 198 198 340 340 340 F F F
214772024-02-04T14:09:40.714Z 25 500-519 198 198 198 339 339 339 F F F
214782024-02-04T14:09:40.714Z 26 520-539 196 196 196 335 335 335 F F F
214792024-02-04T14:09:40.714Z 27 540-559 196 196 196 335 335 335 F F F
214802024-02-04T14:09:40.715Z 28 560-579 194 194 194 330 330 330 F F F
214812024-02-04T14:09:40.715Z 29 580-599 192 192 192 324 324 324 F F F
214822024-02-04T14:09:40.715ZMax gen: 199, Max flush: 341
214832024-02-04T14:09:40.715Z9:26 Test duration
214842024-02-04T14:09:40.715ZTest completed
214852024-02-04T14:09:40.720Z
214862024-02-04T14:09:40.721Zreal 9:25.872187975
214872024-02-04T14:09:40.721Zuser 2:43.826357217
214882024-02-04T14:09:40.721Zsys 28.283762547
214892024-02-04T14:09:40.721Ztrap 0.134988980
214902024-02-04T14:09:40.721Ztflt 0.082369483
214912024-02-04T14:09:40.721Zdflt 0.114015604
214922024-02-04T14:09:40.721Zkflt 0.069096069
214932024-02-04T14:09:40.721Zlock 7:19:57.258566015
214942024-02-04T14:09:40.721Zslp 44:19.840761772
214952024-02-04T14:09:40.721Zlat 9.999368357
214962024-02-04T14:09:40.721Zstop 0.065130368
214972024-02-04T14:09:40.721Zprocess exited: duration 579750 ms, exit code 0
 
214982024-02-04T14:09:40.736Zfound 4 output files
214992024-02-04T14:09:40.736Zuploading: /tmp/test_repair_ds8810.txt (110074 bytes)
215002024-02-04T14:09:41.758Zuploaded: /tmp/test_repair_ds8810.txt
215012024-02-04T14:09:41.764Zuploading: /tmp/test_repair_ds8820.txt (20895 bytes)
215022024-02-04T14:09:42.776Zuploaded: /tmp/test_repair_ds8820.txt
215032024-02-04T14:09:42.777Zuploading: /tmp/test_repair_ds8830.txt (29394 bytes)
215042024-02-04T14:09:43.789Zuploaded: /tmp/test_repair_ds8830.txt
215052024-02-04T14:09:43.789Zuploading: /tmp/test_repair_out.txt (87832 bytes)
215062024-02-04T14:09:44.803Zuploaded: /tmp/test_repair_out.txt