01HNQRV7D30RHQ0J4HJB8GCB0V: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HNQRVWVMRTJMJBS0BGDZ9RDB

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-03T15:49:03.349Zjob dependencies complete; ready to run (waiting for 29 m 12 s)
22024-02-03T17:21:26.236Zjob assigned to worker 01HNQZRPXW5H8AQRB17678AHMJ (queued for 1 h 32 m 22 s)
32024-02-03T17:21:26.623Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-03T17:21:39.689Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-03T17:21:39.689Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-03T17:21:51.564Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-03T17:21:51.564Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-03T17:21:58.521Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-03T17:21:58.521Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-03T17:22:06.768Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-03T17:22:06.774Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-03T17:22:06.790Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-03T17:22:06.790Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-03T17:22:06.806Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-03T17:22:06.806Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-03T17:22:06.826Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-03T17:22:06.826Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-03T17:22:06.854Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-03T17:22:06.854Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-03T17:22:06.870Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-03T17:22:06.870Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-03T17:22:06.894Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-03T17:22:06.894Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-03T17:22:06.916Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-03T17:22:06.919Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-03T17:22:16.649Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-03T17:22:16.650Zstarting task 0: "setup"
282024-02-03T17:22:16.663Z++ uname -s
292024-02-03T17:22:16.667Z+ kern=SunOS
302024-02-03T17:22:16.667Z+ case "$kern" in
312024-02-03T17:22:16.667Z+ groupadd -g 12345 build
322024-02-03T17:22:16.670Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-03T17:22:18.773Z+ zfs create -o mountpoint=/work rpool/work
342024-02-03T17:22:19.417Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-03T17:22:19.420Z+ home_fs=zfs
362024-02-03T17:22:19.421Z+ [[ zfs == autofs ]]
372024-02-03T17:22:19.421Z+ mkdir -p /home/build
382024-02-03T17:22:19.426Z+ chown build:build /home/build /work
392024-02-03T17:22:20.427Z+ chmod 0700 /home/build /work
402024-02-03T17:22:20.430Zprocess exited: duration 3780 ms, exit code 0
 
412024-02-03T17:22:20.436Zstarting task 1: "authentication"
422024-02-03T17:22:20.462Zprocess exited: duration 25 ms, exit code 0
 
432024-02-03T17:22:20.476Zstarting task 2: "build"
442024-02-03T17:22:20.483Z+ banner cores
452024-02-03T17:22:20.530Z
462024-02-03T17:22:20.530Z #### #### ##### ###### ####
472024-02-03T17:22:20.530Z # # # # # # # #
482024-02-03T17:22:20.530Z # # # # # ##### ####
492024-02-03T17:22:20.531Z # # # ##### # #
502024-02-03T17:22:20.531Z # # # # # # # # #
512024-02-03T17:22:20.531Z #### #### # # ###### ####
522024-02-03T17:22:20.531Z
532024-02-03T17:22:20.541Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-03T17:22:20.542Z+ echo 'input bins dir contains:'
552024-02-03T17:22:20.542Zinput bins dir contains:
562024-02-03T17:22:20.542Z+ ls -ltr /input/build/work/bins
572024-02-03T17:22:20.548Ztotal 762219
582024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 121487724 Feb 3 17:21 crucible-downstairs.gz
592024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 91386379 Feb 3 17:21 crucible-hammer.gz
602024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 105093087 Feb 3 17:21 crutest.gz
612024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 72023915 Feb 3 17:22 dsc.gz
622024-02-03T17:22:20.548Z+ echo 'input script dir contains:'
632024-02-03T17:22:20.548Zinput script dir contains:
642024-02-03T17:22:20.548Z+ ls -ltr /input/build/work/scripts
652024-02-03T17:22:20.548Ztotal 38
662024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 3056 Feb 3 17:22 perf-downstairs-tick.d
672024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 939 Feb 3 17:22 test_ds.sh
682024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 3746 Feb 3 17:22 test_live_repair.sh
692024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 6416 Feb 3 17:22 test_repair.sh
702024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 2969 Feb 3 17:22 test_replay.sh
712024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 9465 Feb 3 17:22 test_up.sh
722024-02-03T17:22:20.548Z-rw-r--r-- 1 root root 2824 Feb 3 17:22 upstairs_info.d
732024-02-03T17:22:20.548Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-03T17:22:20.576Z+ echo ' chmod input script dir contains:'
752024-02-03T17:22:20.576Z chmod input script dir contains:
762024-02-03T17:22:20.576Z+ ls -ltr /input/build/work/scripts
772024-02-03T17:22:20.576Ztotal 38
782024-02-03T17:22:20.576Z-rwxr-xr-x 1 root root 3056 Feb 3 17:22 perf-downstairs-tick.d
792024-02-03T17:22:20.576Z-rwxr-xr-x 1 root root 939 Feb 3 17:22 test_ds.sh
802024-02-03T17:22:20.576Z-rwxr-xr-x 1 root root 3746 Feb 3 17:22 test_live_repair.sh
812024-02-03T17:22:20.576Z-rwxr-xr-x 1 root root 6416 Feb 3 17:22 test_repair.sh
822024-02-03T17:22:20.576Z-rwxr-xr-x 1 root root 2969 Feb 3 17:22 test_replay.sh
832024-02-03T17:22:20.576Z-rwxr-xr-x 1 root root 9465 Feb 3 17:22 test_up.sh
842024-02-03T17:22:20.576Z-rwxr-xr-x 1 root root 2824 Feb 3 17:22 upstairs_info.d
852024-02-03T17:22:20.576Z+ banner unpack
862024-02-03T17:22:20.576Z
872024-02-03T17:22:20.577Z # # # # ##### ## #### # #
882024-02-03T17:22:20.577Z # # ## # # # # # # # # #
892024-02-03T17:22:20.577Z # # # # # # # # # # ####
902024-02-03T17:22:20.577Z # # # # # ##### ###### # # #
912024-02-03T17:22:20.577Z # # # ## # # # # # # #
922024-02-03T17:22:20.577Z #### # # # # # #### # #
932024-02-03T17:22:20.577Z
942024-02-03T17:22:20.577Z+ mkdir -p /var/tmp/bins
952024-02-03T17:22:20.577Z+ for t in "$input/bins/"*.gz
962024-02-03T17:22:20.577Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-03T17:22:20.577Z+ b=crucible-downstairs.gz
982024-02-03T17:22:20.577Z+ b=crucible-downstairs
992024-02-03T17:22:20.577Z+ gunzip
1002024-02-03T17:22:24.993Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-03T17:22:24.996Z+ for t in "$input/bins/"*.gz
1022024-02-03T17:22:24.997Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-03T17:22:25.001Z+ b=crucible-hammer.gz
1042024-02-03T17:22:25.001Z+ b=crucible-hammer
1052024-02-03T17:22:25.001Z+ gunzip
1062024-02-03T17:22:28.158Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-03T17:22:28.161Z+ for t in "$input/bins/"*.gz
1082024-02-03T17:22:28.164Z++ basename /input/build/work/bins/crutest.gz
1092024-02-03T17:22:28.164Z+ b=crutest.gz
1102024-02-03T17:22:28.165Z+ b=crutest
1112024-02-03T17:22:28.165Z+ gunzip
1122024-02-03T17:22:31.853Z+ chmod +x /var/tmp/bins/crutest
1132024-02-03T17:22:31.856Z+ for t in "$input/bins/"*.gz
1142024-02-03T17:22:31.859Z++ basename /input/build/work/bins/dsc.gz
1152024-02-03T17:22:31.859Z+ b=dsc.gz
1162024-02-03T17:22:31.859Z+ b=dsc
1172024-02-03T17:22:31.859Z+ gunzip
1182024-02-03T17:22:34.406Z+ chmod +x /var/tmp/bins/dsc
1192024-02-03T17:22:34.412Z+ export BINDIR=/var/tmp/bins
1202024-02-03T17:22:34.412Z+ BINDIR=/var/tmp/bins
1212024-02-03T17:22:34.412Z+ banner repair
1222024-02-03T17:22:34.412Z
1232024-02-03T17:22:34.412Z ##### ###### ##### ## # #####
1242024-02-03T17:22:34.412Z # # # # # # # # # #
1252024-02-03T17:22:34.412Z # # ##### # # # # # # #
1262024-02-03T17:22:34.412Z ##### # ##### ###### # #####
1272024-02-03T17:22:34.412Z # # # # # # # # #
1282024-02-03T17:22:34.412Z # # ###### # # # # # #
1292024-02-03T17:22:34.412Z
1302024-02-03T17:22:34.413Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-03T17:22:34.510ZTurn off color for downstairs dump
1322024-02-03T17:22:34.537ZCreating region directory at: /var/tmp/test_repair
1332024-02-03T17:22:34.541ZCreating dsc directory at: /tmp/dsc
1342024-02-03T17:22:35.037ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.49849117
1352024-02-03T17:22:35.178ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.14156443
1362024-02-03T17:22:35.325ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.14680053
1372024-02-03T17:22:35.325ZRegion set with 3 regions was created
1382024-02-03T17:22:35.345Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1131 1132 1133
1392024-02-03T17:22:35.346ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-03T17:22:35.443Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.595110853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135}
1412024-02-03T17:22:35.443Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.596958972Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135}
1422024-02-03T17:22:35.443Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.596990476Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135}
1432024-02-03T17:22:35.443Z{"msg":"Crucible 7ce9ac5f-3541-4874-a937-8079b3158f6d has session id: 21aeba7f-0ff4-4525-b465-f511ee1d993f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.597012563Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1442024-02-03T17:22:35.448Z{"msg":"Upstairs opts: Upstairs UUID: 7ce9ac5f-3541-4874-a937-8079b3158f6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.597043578Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1452024-02-03T17:22:35.448Z{"msg":"Crucible stats registered with UUID: 7ce9ac5f-3541-4874-a937-8079b3158f6d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.597087746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1462024-02-03T17:22:35.448ZCrucible runtime is spawned
1472024-02-03T17:22:35.449Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.599887419Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135}
1482024-02-03T17:22:35.449Z{"msg":"7ce9ac5f-3541-4874-a937-8079b3158f6d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.600178561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1492024-02-03T17:22:35.449Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.600232255Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1502024-02-03T17:22:35.449Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.600402104Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1512024-02-03T17:22:35.449Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.600508138Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1522024-02-03T17:22:35.449Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.60060345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1532024-02-03T17:22:35.449Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.600680708Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1542024-02-03T17:22:35.449Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.600739693Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1552024-02-03T17:22:35.450Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.60084967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1562024-02-03T17:22:35.450Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.600898305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1572024-02-03T17:22:35.450Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.601449426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1582024-02-03T17:22:35.450Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.601566151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1592024-02-03T17:22:35.450Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.601660115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1602024-02-03T17:22:35.450Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.601764459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1612024-02-03T17:22:35.456Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.605523561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1622024-02-03T17:22:35.456Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.605587496Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1632024-02-03T17:22:35.456Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.60563473Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1642024-02-03T17:22:35.456Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.60565996Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1652024-02-03T17:22:35.456Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.605681838Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1662024-02-03T17:22:35.456Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.605770028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1672024-02-03T17:22:35.456Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.608395857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1682024-02-03T17:22:35.457Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.608479117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1692024-02-03T17:22:35.457Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.60865879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1702024-02-03T17:22:35.462Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.610390258Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1712024-02-03T17:22:35.462Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.610488364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1722024-02-03T17:22:35.462Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.610538903Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1732024-02-03T17:22:35.462Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.610562815Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1742024-02-03T17:22:35.462Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.610604885Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1752024-02-03T17:22:35.462Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.610629498Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1762024-02-03T17:22:35.462Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.610650016Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1772024-02-03T17:22:35.462Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.610687191Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1782024-02-03T17:22:35.462Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.611338787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1792024-02-03T17:22:35.462Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.611404185Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1802024-02-03T17:22:35.462Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612568314Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1812024-02-03T17:22:35.463Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612646611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1822024-02-03T17:22:35.463Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612694889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1832024-02-03T17:22:35.463Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612721451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1842024-02-03T17:22:35.463Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612773143Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1852024-02-03T17:22:35.463Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612799258Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1862024-02-03T17:22:35.463Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612842305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1872024-02-03T17:22:35.463Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612866212Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1882024-02-03T17:22:35.463Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.61290752Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1892024-02-03T17:22:35.463Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.612931365Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1902024-02-03T17:22:35.463Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.613095125Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1912024-02-03T17:22:35.463Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.613150608Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1922024-02-03T17:22:35.463Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.613194694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1932024-02-03T17:22:35.463ZWait for a query_work_queue command to finish before sending IO
1942024-02-03T17:22:35.464Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.613313646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1952024-02-03T17:22:35.464Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.613364989Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1962024-02-03T17:22:35.464Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.613390732Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1972024-02-03T17:22:35.464Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.61343823Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1982024-02-03T17:22:35.464Z{"msg":"7ce9ac5f-3541-4874-a937-8079b3158f6d is now active with session: 21aeba7f-0ff4-4525-b465-f511ee1d993f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.613463855Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
1992024-02-03T17:22:35.464Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.61350385Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135,"session_id":"21aeba7f-0ff4-4525-b465-f511ee1d993f"}
2002024-02-03T17:22:35.464Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:34.61352665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135}
2012024-02-03T17:22:35.464ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-03T17:22:35.464ZFill test
2032024-02-03T17:22:35.748ZRead and Verify all blocks (0..600 range:false)
2042024-02-03T17:22:35.866ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-03T17:22:35.866ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-03T17:22:35.874Z----------------------------------------------------------------
2072024-02-03T17:22:35.874Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-03T17:22:35.875ZStates: Active Active Active
2092024-02-03T17:22:35.875ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-03T17:22:35.875Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-03T17:22:35.875Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-03T17:22:35.875Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-03T17:22:35.875Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-03T17:22:35.875Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-03T17:22:35.875Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-03T17:22:35.875Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-03T17:22:35.875Z New 0 0 0 0
2182024-02-03T17:22:35.875Z Sent 0 0 0 0
2192024-02-03T17:22:35.875Z Done 6 6 6 18
2202024-02-03T17:22:35.875Z Skipped 0 0 0 0
2212024-02-03T17:22:35.875Z Error 0 0 0 0
2222024-02-03T17:22:35.875ZLast Flush: 1006 1006 1006
2232024-02-03T17:22:35.875ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-03T17:22:35.875ZUpstairs last five completed: 13 12 11 10 9
2252024-02-03T17:22:35.875ZCLIENT: Up:0 ds:6 act:3
2262024-02-03T17:22:39.861Z----------------------------------------------------------------
2272024-02-03T17:22:39.861Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-03T17:22:39.861ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-03T17:22:39.861ZUpstairs last five completed: 14 13 12 11 10
2302024-02-03T17:22:39.861ZCLIENT: Up:0 ds:0 act:3
2312024-02-03T17:22:39.861ZCLIENT: All crucible jobs finished, exiting program
2322024-02-03T17:22:39.864Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:22:39.016154484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1135}
2332024-02-03T17:22:39.872Z
2342024-02-03T17:22:39.872ZBegin loop 1
2352024-02-03T17:22:39.872ZDownstairs to restart: 0
2362024-02-03T17:22:39.902Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.057194942Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137}
2372024-02-03T17:22:39.906Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.057690862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137}
2382024-02-03T17:22:39.906Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.057804248Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137}
2392024-02-03T17:22:39.906Z{"msg":"Crucible 80ea5145-e9f0-4ae9-82d1-8b99a8ac747b has session id: 0dd69b2f-dfed-40e0-9871-ec9029415e8f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.057896795Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2402024-02-03T17:22:39.906Z{"msg":"Upstairs opts: Upstairs UUID: 80ea5145-e9f0-4ae9-82d1-8b99a8ac747b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.057937505Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2412024-02-03T17:22:39.906Z{"msg":"Crucible stats registered with UUID: 80ea5145-e9f0-4ae9-82d1-8b99a8ac747b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.05799285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2422024-02-03T17:22:39.906ZCrucible runtime is spawned
2432024-02-03T17:22:39.906Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058192998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137}
2442024-02-03T17:22:39.906Z{"msg":"80ea5145-e9f0-4ae9-82d1-8b99a8ac747b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058393026Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2452024-02-03T17:22:39.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058452393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2462024-02-03T17:22:39.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058479595Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2472024-02-03T17:22:39.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058548411Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2482024-02-03T17:22:39.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058588898Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2492024-02-03T17:22:39.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058614554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2502024-02-03T17:22:39.907Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058636878Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2512024-02-03T17:22:39.907Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058658546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2522024-02-03T17:22:39.907Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058688173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2532024-02-03T17:22:39.907Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.058840433Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2542024-02-03T17:22:39.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.05897761Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2552024-02-03T17:22:39.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.059064335Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2562024-02-03T17:22:39.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.059163892Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2572024-02-03T17:22:39.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.060488606Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2582024-02-03T17:22:39.907Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.060587266Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2592024-02-03T17:22:39.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.06063194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2602024-02-03T17:22:39.907Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.06067398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2612024-02-03T17:22:39.911Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.061486037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2622024-02-03T17:22:39.911Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.061547474Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2632024-02-03T17:22:39.911Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.061695638Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2642024-02-03T17:22:39.911Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.06178122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2652024-02-03T17:22:39.911Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.062662994Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2662024-02-03T17:22:39.911Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.062726735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2672024-02-03T17:22:39.911Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.062805368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2682024-02-03T17:22:39.911Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.062852375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2692024-02-03T17:22:39.912Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.062877824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2702024-02-03T17:22:39.912Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.062900845Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2712024-02-03T17:22:39.912Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.062951953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2722024-02-03T17:22:39.912Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063502122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2732024-02-03T17:22:39.912Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063590279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2742024-02-03T17:22:39.912Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063638526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2752024-02-03T17:22:39.912Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063694744Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2762024-02-03T17:22:39.912Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063741857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2772024-02-03T17:22:39.912Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063800152Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2782024-02-03T17:22:39.912Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063845454Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2792024-02-03T17:22:39.912Z{"msg":"Wait for a query_work_queue command to finish before sending IO
2802024-02-03T17:22:39.912Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063906597Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2812024-02-03T17:22:39.912Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.063951514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2822024-02-03T17:22:39.912Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064016668Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2832024-02-03T17:22:39.912Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064104416Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2842024-02-03T17:22:39.912Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064132317Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2852024-02-03T17:22:39.912Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064154823Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2862024-02-03T17:22:39.912Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064206091Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2872024-02-03T17:22:39.912Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.06423152Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2882024-02-03T17:22:39.912Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064262699Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2892024-02-03T17:22:39.912Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064312569Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2902024-02-03T17:22:39.912Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064336728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2912024-02-03T17:22:39.912Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064389021Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2922024-02-03T17:22:39.912Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064414799Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2932024-02-03T17:22:39.913Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064462076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2942024-02-03T17:22:39.913Z{"msg":"80ea5145-e9f0-4ae9-82d1-8b99a8ac747b is now active with session: 0dd69b2f-dfed-40e0-9871-ec9029415e8f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064488822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2952024-02-03T17:22:39.913Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064533396Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137,"session_id":"0dd69b2f-dfed-40e0-9871-ec9029415e8f"}
2962024-02-03T17:22:39.913Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:39.064556268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137}
2972024-02-03T17:22:39.913ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2982024-02-03T17:22:39.913ZLoading write count information from file "/tmp/test_repair_verify.data"
2992024-02-03T17:22:39.913ZRun Repair workload
3002024-02-03T17:22:39.916Z01/30 Write block 559 len 8192 data: 2 2
3012024-02-03T17:22:39.916Z02/30 Write block 169 len 28672 data: 2 2 2 2 2 2 2
3022024-02-03T17:22:39.916Z03/30 Read block 40 len 36864
3032024-02-03T17:22:39.919Z04/30 Read block 364 len 36864
3042024-02-03T17:22:39.922Z05/30 Write block 551 len 24576 data: 2 2 2 2 2 2
3052024-02-03T17:22:39.922Z06/30 Read block 234 len 4096
3062024-02-03T17:22:39.922Z07/30 Read block 437 len 8192
3072024-02-03T17:22:39.926Z08/30 Write block 275 len 12288 data: 2 2 2
3082024-02-03T17:22:39.926Z09/30 Write block 221 len 8192 data: 2 2
3092024-02-03T17:22:39.926Z10/30 Write block 362 len 28672 data: 2 2 2 2 2 2 2
3102024-02-03T17:22:39.926Z11/30 Read block 540 len 16384
3112024-02-03T17:22:39.930Z12/30 Write block 393 len 16384 data: 2 2 2 2
3122024-02-03T17:22:39.930Z13/30 Read block 495 len 16384
3132024-02-03T17:22:39.937Z14/30 Read block 444 len 24576
3142024-02-03T17:22:39.937Z15/30 Read block 545 len 8192
3152024-02-03T17:22:39.937Z16/30 Read block 291 len 32768
3162024-02-03T17:22:39.937Z17/30 Read block 122 len 16384
3172024-02-03T17:22:39.937Z18/30 Read block 234 len 4096
3182024-02-03T17:22:39.937Z19/30 Read block 438 len 32768
3192024-02-03T17:22:39.941Z20/30 Read block 70 len 4096
3202024-02-03T17:22:39.941Z21/30 Write block 193 len 4096 data: 2
3212024-02-03T17:22:39.942Z22/30 Read block 249 len 36864
3222024-02-03T17:22:39.945Z23/30 Write block 593 len 24576 data: 2 2 2 2 2 2
3232024-02-03T17:22:39.945Z24/30 Write block 448 len 24576 data: 2 2 2 2 2 2
3242024-02-03T17:22:39.945Z25/30 Write block 130 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3252024-02-03T17:22:39.948Z26/30 Write block 447 len 32768 data: 2 3 3 3 3 3 3 2
3262024-02-03T17:22:39.948Z27/30 Write block 281 len 36864 data: 2 2 2 2 2 2 2 2 2
3272024-02-03T17:22:39.951Z28/30 Read block 490 len 8192
3282024-02-03T17:22:39.951Z29/30 Read block 517 len 4096
3292024-02-03T17:22:39.951Z30/30 Read block 410 len 20480
3302024-02-03T17:22:39.955Z----------------------------------------------------------------
3312024-02-03T17:22:39.955Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3322024-02-03T17:22:39.955ZStates: Active Active Active
3332024-02-03T17:22:39.955ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3342024-02-03T17:22:39.955Z 1 Acked 1000 Write 2 Sent Done Done false
3352024-02-03T17:22:39.955Z 2 Acked 1001 Write 7 Sent Done Done false
3362024-02-03T17:22:39.955Z 3 Acked 1002 Read 9 Sent Done Done false
3372024-02-03T17:22:39.955Z 4 Acked 1003 Read 9 Sent Done Done false
3382024-02-03T17:22:39.955Z 5 Acked 1004 Write 6 Sent Done Done false
3392024-02-03T17:22:39.955Z 6 Acked 1005 Read 1 Sent Done Done false
3402024-02-03T17:22:39.955Z 7 Acked 1006 Read 2 Sent Done Done false
3412024-02-03T17:22:39.955Z 8 Acked 1007 Write 3 Sent Done Done false
3422024-02-03T17:22:39.955Z 9 Acked 1008 Write 2 Sent Done Done false
3432024-02-03T17:22:39.955Z 10 Acked 1009 Write 7 Sent Done Done false
3442024-02-03T17:22:39.955Z 11 Acked 1010 Read 4 Sent Done Done false
3452024-02-03T17:22:39.955Z 12 Acked 1011 Write 4 Sent Done Done false
3462024-02-03T17:22:39.955Z 13 Acked 1012 Read 4 Sent Done Done false
3472024-02-03T17:22:39.956Z 14 Acked 1013 Read 6 Sent Done Done false
3482024-02-03T17:22:39.956Z 15 Acked 1014 Read 2 Sent Done Done false
3492024-02-03T17:22:39.956Z 16 Acked 1015 Read 8 Sent Done Done false
3502024-02-03T17:22:39.956Z 17 Acked 1016 Read 4 Sent Done Done false
3512024-02-03T17:22:39.956Z 18 Acked 1017 Read 1 Sent Done Done false
3522024-02-03T17:22:39.956Z 19 Acked 1018 Read 8 Sent Done Done false
3532024-02-03T17:22:39.956Z 20 Acked 1019 Read 1 Sent Done Done false
3542024-02-03T17:22:39.956Z 21 Acked 1020 Write 1 Sent Done Done false
3552024-02-03T17:22:39.956Z 22 Acked 1021 Read 9 Sent Done Done false
3562024-02-03T17:22:39.956Z 23 Acked 1022 Write 6 Sent Done Done false
3572024-02-03T17:22:39.956Z 24 Acked 1023 Write 6 Sent Done Done false
3582024-02-03T17:22:39.956Z 25 Acked 1024 Write 10 Sent Done Done false
3592024-02-03T17:22:39.956Z 26 Acked 1025 Write 8 Sent Done Done false
3602024-02-03T17:22:39.956Z 27 Acked 1026 Write 9 Sent Done Done false
3612024-02-03T17:22:39.956Z 28 Acked 1027 Read 2 Sent Done Done false
3622024-02-03T17:22:39.956Z 29 Acked 1028 Read 1 Sent Done Done false
3632024-02-03T17:22:39.956Z 30 Acked 1029 Read 5 Sent Sent Done false
3642024-02-03T17:22:39.956Z STATES DS:0 DS:1 DS:2 TOTAL
3652024-02-03T17:22:39.956Z New 0 0 0 0
3662024-02-03T17:22:39.956Z Sent 30 1 0 31
3672024-02-03T17:22:39.956Z Done 0 29 30 59
3682024-02-03T17:22:39.956Z Skipped 0 0 0 0
3692024-02-03T17:22:39.956Z Error 0 0 0 0
3702024-02-03T17:22:39.956ZLast Flush: 0 0 0
3712024-02-03T17:22:39.956ZDownstairs last five completed:
3722024-02-03T17:22:39.957ZUpstairs last five completed: 30 29 28 27 26
3732024-02-03T17:22:39.957Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:22:39.108737341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1137}
3742024-02-03T17:22:39.957ZWrote out file "/tmp/test_repair_verify.data"
3752024-02-03T17:22:39.961Z
3762024-02-03T17:22:39.968ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3772024-02-03T17:22:39.999ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3782024-02-03T17:22:40.003Z 0 000-019 1 1 1 1 1 1 F F F
3792024-02-03T17:22:40.003Z 1 020-039 1 1 1 1 1 1 F F F
3802024-02-03T17:22:40.003Z 2 040-059 1 1 1 1 1 1 F F F
3812024-02-03T17:22:40.003Z 3 060-079 1 1 1 1 1 1 F F F
3822024-02-03T17:22:40.003Z 4 080-099 1 1 1 1 1 1 F F F
3832024-02-03T17:22:40.003Z 5 100-119 1 1 1 1 1 1 F F F
3842024-02-03T17:22:40.003Z 6 120-139 1 1 1 1 1 1 F T T <---
3852024-02-03T17:22:40.003Z 7 140-159 1 1 1 1 1 1 F F F
3862024-02-03T17:22:40.003Z 8 160-179 1 1 1 1 1 1 F T T <---
3872024-02-03T17:22:40.003Z 9 180-199 1 1 1 1 1 1 F T T <---
3882024-02-03T17:22:40.003Z 10 200-219 1 1 1 1 1 1 F F F
3892024-02-03T17:22:40.003Z 11 220-239 1 1 1 1 1 1 F T T <---
3902024-02-03T17:22:40.003Z 12 240-259 1 1 1 1 1 1 F F F
3912024-02-03T17:22:40.003Z 13 260-279 1 1 1 1 1 1 F T T <---
3922024-02-03T17:22:40.003Z 14 280-299 1 1 1 1 1 1 F T T <---
3932024-02-03T17:22:40.003Z 15 300-319 1 1 1 1 1 1 F F F
3942024-02-03T17:22:40.003Z 16 320-339 1 1 1 1 1 1 F F F
3952024-02-03T17:22:40.003Z 17 340-359 1 1 1 1 1 1 F F F
3962024-02-03T17:22:40.003Z 18 360-379 1 1 1 1 1 1 F T T <---
3972024-02-03T17:22:40.003Z 19 380-399 1 1 1 1 1 1 F T T <---
3982024-02-03T17:22:40.003Z 20 400-419 1 1 1 1 1 1 F F F
3992024-02-03T17:22:40.003Z 21 420-439 1 1 1 1 1 1 F F F
4002024-02-03T17:22:40.004Z 22 440-459 1 1 1 1 1 1 F T T <---
4012024-02-03T17:22:40.004Z 23 460-479 1 1 1 1 1 1 F F F
4022024-02-03T17:22:40.004Z 24 480-499 1 1 1 1 1 1 F F F
4032024-02-03T17:22:40.004Z 25 500-519 1 1 1 1 1 1 F F F
4042024-02-03T17:22:40.004Z 26 520-539 1 1 1 1 1 1 F F F
4052024-02-03T17:22:40.004Z 27 540-559 1 1 1 1 1 1 F T T <---
4062024-02-03T17:22:40.004Z 28 560-579 1 1 1 1 1 1 F T T <---
4072024-02-03T17:22:40.004Z 29 580-599 1 1 1 1 1 1 F T T <---
4082024-02-03T17:22:40.004ZMax gen: 1, Max flush: 1
4092024-02-03T17:22:40.004ZError: Difference in extent metadata found!
4102024-02-03T17:22:40.011ZA Difference in extent metadata is expected here
4112024-02-03T17:22:40.011ZOn loop 1
4122024-02-03T17:22:40.011Z
4132024-02-03T17:22:40.017ZVerifying data now
4142024-02-03T17:22:45.137ZNo change after verify
4152024-02-03T17:22:45.137ZLoop: 1 Downstairs dump after verify (and repair):
4162024-02-03T17:22:45.161ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4172024-02-03T17:22:45.161Z 0 000-019 1 1 1 1 1 1 F F F
4182024-02-03T17:22:45.167Z 1 020-039 1 1 1 1 1 1 F F F
4192024-02-03T17:22:45.167Z 2 040-059 1 1 1 1 1 1 F F F
4202024-02-03T17:22:45.167Z 3 060-079 1 1 1 1 1 1 F F F
4212024-02-03T17:22:45.167Z 4 080-099 1 1 1 1 1 1 F F F
4222024-02-03T17:22:45.167Z 5 100-119 1 1 1 1 1 1 F F F
4232024-02-03T17:22:45.167Z 6 120-139 2 2 2 2 2 2 F F F
4242024-02-03T17:22:45.167Z 7 140-159 1 1 1 1 1 1 F F F
4252024-02-03T17:22:45.168Z 8 160-179 2 2 2 2 2 2 F F F
4262024-02-03T17:22:45.168Z 9 180-199 2 2 2 2 2 2 F F F
4272024-02-03T17:22:45.168Z 10 200-219 1 1 1 1 1 1 F F F
4282024-02-03T17:22:45.168Z 11 220-239 2 2 2 2 2 2 F F F
4292024-02-03T17:22:45.168Z 12 240-259 1 1 1 1 1 1 F F F
4302024-02-03T17:22:45.168Z 13 260-279 2 2 2 2 2 2 F F F
4312024-02-03T17:22:45.168Z 14 280-299 2 2 2 2 2 2 F F F
4322024-02-03T17:22:45.168Z 15 300-319 1 1 1 1 1 1 F F F
4332024-02-03T17:22:45.168Z 16 320-339 1 1 1 1 1 1 F F F
4342024-02-03T17:22:45.168Z 17 340-359 1 1 1 1 1 1 F F F
4352024-02-03T17:22:45.168Z 18 360-379 2 2 2 2 2 2 F F F
4362024-02-03T17:22:45.168Z 19 380-399 2 2 2 2 2 2 F F F
4372024-02-03T17:22:45.168Z 20 400-419 1 1 1 1 1 1 F F F
4382024-02-03T17:22:45.168Z 21 420-439 1 1 1 1 1 1 F F F
4392024-02-03T17:22:45.168Z 22 440-459 2 2 2 2 2 2 F F F
4402024-02-03T17:22:45.168Z 23 460-479 1 1 1 1 1 1 F F F
4412024-02-03T17:22:45.168Z 24 480-499 1 1 1 1 1 1 F F F
4422024-02-03T17:22:45.168Z 25 500-519 1 1 1 1 1 1 F F F
4432024-02-03T17:22:45.168Z 26 520-539 1 1 1 1 1 1 F F F
4442024-02-03T17:22:45.168Z 27 540-559 2 2 2 2 2 2 F F F
4452024-02-03T17:22:45.168Z 28 560-579 2 2 2 2 2 2 F F F
4462024-02-03T17:22:45.168Z 29 580-599 2 2 2 2 2 2 F F F
4472024-02-03T17:22:45.168ZMax gen: 2, Max flush: 2
4482024-02-03T17:22:45.172Z
4492024-02-03T17:22:45.172ZBegin loop 2
4502024-02-03T17:22:45.172ZDownstairs to restart: 2
4512024-02-03T17:22:45.198Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.353020179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145}
4522024-02-03T17:22:45.198Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.353562528Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145}
4532024-02-03T17:22:45.202Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.353634824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","Crucible runtime is spawned
4542024-02-03T17:22:45.202Zpid":1145}
4552024-02-03T17:22:45.202Z{"msg":"Crucible b0cdd03e-60e4-4827-b205-0568be462198 has session id: 84f54815-e1c3-4505-9daf-c1a07824ae16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.353765859Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4562024-02-03T17:22:45.202Z{"msg":"Upstairs opts: Upstairs UUID: b0cdd03e-60e4-4827-b205-0568be462198, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.353814341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4572024-02-03T17:22:45.202Z{"msg":"Crucible stats registered with UUID: b0cdd03e-60e4-4827-b205-0568be462198","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.353866632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4582024-02-03T17:22:45.202Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.353891518Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145}
4592024-02-03T17:22:45.202Z{"msg":"b0cdd03e-60e4-4827-b205-0568be462198 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.35409779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4602024-02-03T17:22:45.203Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354149216Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4612024-02-03T17:22:45.203Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354190156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4622024-02-03T17:22:45.203Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354235097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4632024-02-03T17:22:45.203Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354277524Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4642024-02-03T17:22:45.203Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354359333Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4652024-02-03T17:22:45.203Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.35440336Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4662024-02-03T17:22:45.203Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354446508Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4672024-02-03T17:22:45.203Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.35450331Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4682024-02-03T17:22:45.203Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354572177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4692024-02-03T17:22:45.203Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354615352Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4702024-02-03T17:22:45.203Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.354706304Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4712024-02-03T17:22:45.203Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.35488037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4722024-02-03T17:22:45.203Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.356028114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4732024-02-03T17:22:45.203Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.356117611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4742024-02-03T17:22:45.203Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.356167313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4752024-02-03T17:22:45.203Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.356239144Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4762024-02-03T17:22:45.203Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.356859607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4772024-02-03T17:22:45.203Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.356958697Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4782024-02-03T17:22:45.203Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.356988783Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4792024-02-03T17:22:45.203Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.357093297Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4802024-02-03T17:22:45.208Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.357888435Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4812024-02-03T17:22:45.208Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.35818792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4822024-02-03T17:22:45.208Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358255785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4832024-02-03T17:22:45.208Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358282427Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4842024-02-03T17:22:45.208Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358340625Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4852024-02-03T17:22:45.208Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358368565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4862024-02-03T17:22:45.208Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358410895Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4872024-02-03T17:22:45.208Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358725053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4882024-02-03T17:22:45.208Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358810157Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4892024-02-03T17:22:45.208Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358842322Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4902024-02-03T17:22:45.208Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358886321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4912024-02-03T17:22:45.208Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358912775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4922024-02-03T17:22:45.208Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358954162Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4932024-02-03T17:22:45.208Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.358978762Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4942024-02-03T17:22:45.209Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359002098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4952024-02-03T17:22:45.209ZWait for a query_work_queue command to finish before sending IO
4962024-02-03T17:22:45.209Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359065611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4972024-02-03T17:22:45.209Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.35912268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4982024-02-03T17:22:45.209Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359150544Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
4992024-02-03T17:22:45.209Z{"msg":"Max found gen is 3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359192127Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5002024-02-03T17:22:45.209Z{"msg":"Generation requested: 4 >= found:3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359215842Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5012024-02-03T17:22:45.209Z{"msg":"Next flush: 3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359256085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5022024-02-03T17:22:45.209Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359279821Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5032024-02-03T17:22:45.209Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.3593117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5042024-02-03T17:22:45.209Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359376295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5052024-02-03T17:22:45.209Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359403979Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5062024-02-03T17:22:45.209Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359469474Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5072024-02-03T17:22:45.209Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359498225Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5082024-02-03T17:22:45.209Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359564881Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5092024-02-03T17:22:45.209Z{"msg":"b0cdd03e-60e4-4827-b205-0568be462198 is now active with session: 84f54815-e1c3-4505-9daf-c1a07824ae16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359599075Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5102024-02-03T17:22:45.209Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359663001Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145,"session_id":"84f54815-e1c3-4505-9daf-c1a07824ae16"}
5112024-02-03T17:22:45.209ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5122024-02-03T17:22:45.209Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:44.359738614Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145}
5132024-02-03T17:22:45.209ZLoading write count information from file "/tmp/test_repair_verify.data"
5142024-02-03T17:22:45.210ZRun Repair workload
5152024-02-03T17:22:45.210Z01/30 Write block 199 len 8192 data: 2 2
5162024-02-03T17:22:45.213Z02/30 Read block 512 len 40960
5172024-02-03T17:22:45.214Z03/30 Read block 193 len 32768
5182024-02-03T17:22:45.214Z04/30 Read block 41 len 32768
5192024-02-03T17:22:45.217Z05/30 Read block 123 len 24576
5202024-02-03T17:22:45.217Z06/30 Flush
5212024-02-03T17:22:45.222Z07/30 Write block 8 len 4096 data: 2
5222024-02-03T17:22:45.222Z08/30 Write block 488 len 4096 data: 2
5232024-02-03T17:22:45.222Z09/30 Flush
5242024-02-03T17:22:45.222Z10/30 Write block 456 len 28672 data: 2 2 2 2 2 2 2
5252024-02-03T17:22:45.226Z11/30 Read block 478 len 28672
5262024-02-03T17:22:45.226Z12/30 Read block 238 len 4096
5272024-02-03T17:22:45.229Z13/30 Read block 110 len 20480
5282024-02-03T17:22:45.230Z14/30 Read block 488 len 28672
5292024-02-03T17:22:45.233Z15/30 Write block 591 len 36864 data: 2 2 3 3 3 3 3 3 2
5302024-02-03T17:22:45.233Z16/30 Write block 477 len 28672 data: 2 2 2 2 2 2 2
5312024-02-03T17:22:45.233Z17/30 Read block 55 len 8192
5322024-02-03T17:22:45.237Z18/30 Read block 205 len 24576
5332024-02-03T17:22:45.237Z19/30 Read block 483 len 28672
5342024-02-03T17:22:45.237Z20/30 Read block 498 len 20480
5352024-02-03T17:22:45.240Z21/30 Flush
5362024-02-03T17:22:45.244Z22/30 Write block 505 len 20480 data: 2 2 2 2 2
5372024-02-03T17:22:45.244Z23/30 Write block 416 len 8192 data: 2 2
5382024-02-03T17:22:45.244Z24/30 Flush
5392024-02-03T17:22:45.256Z25/30 Write block 443 len 8192 data: 2 2
5402024-02-03T17:22:45.260Z26/30 Write block 221 len 32768 data: 3 3 2 2 2 2 2 2
5412024-02-03T17:22:45.260Z27/30 Read block 69 len 20480
5422024-02-03T17:22:45.260Z28/30 Read block 94 len 16384
5432024-02-03T17:22:45.260Z29/30 Write block 162 len 36864 data: 2 2 2 2 2 2 2 3 3
5442024-02-03T17:22:45.260Z30/30 Read block 563 len 36864
5452024-02-03T17:22:45.260Z----------------------------------------------------------------
5462024-02-03T17:22:45.260Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5472024-02-03T17:22:45.260ZStates: Active Active Active
5482024-02-03T17:22:45.260ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5492024-02-03T17:22:45.260Z 1 Acked 1000 Write 2 Done Done Done false
5502024-02-03T17:22:45.260Z 2 Acked 1001 Read 10 Done Done Done false
5512024-02-03T17:22:45.260Z 3 Acked 1002 Read 8 Done Done Sent false
5522024-02-03T17:22:45.260Z 4 Acked 1003 Read 8 Done Done Sent false
5532024-02-03T17:22:45.260Z 5 Acked 1004 Read 6 Done Done Sent false
5542024-02-03T17:22:45.260Z 6 Acked 1005 Flush 0 Done Done Sent false
5552024-02-03T17:22:45.260Z 7 Acked 1006 Write 1 Done Done Sent false
5562024-02-03T17:22:45.260Z 8 Acked 1007 Write 1 Done Done Sent false
5572024-02-03T17:22:45.260Z 9 Acked 1008 Flush 0 Done Done Sent false
5582024-02-03T17:22:45.261Z 10 Acked 1009 Write 7 Done Done Sent false
5592024-02-03T17:22:45.261Z 11 Acked 1010 Read 7 Done Done Sent false
5602024-02-03T17:22:45.261Z 12 Acked 1011 Read 1 Done Done Sent false
5612024-02-03T17:22:45.261Z 13 Acked 1012 Read 5 Done Done Sent false
5622024-02-03T17:22:45.261Z 14 Acked 1013 Read 7 Done Done Sent false
5632024-02-03T17:22:45.261Z 15 Acked 1014 Write 9 Done Done Sent false
5642024-02-03T17:22:45.261Z 16 Acked 1015 Write 7 Done Done Sent false
5652024-02-03T17:22:45.261Z 17 Acked 1016 Read 2 Done Done Sent false
5662024-02-03T17:22:45.261Z 18 Acked 1017 Read 6 Done Done Sent false
5672024-02-03T17:22:45.261Z 19 Acked 1018 Read 7 Done Done Sent false
5682024-02-03T17:22:45.261Z 20 Acked 1019 Read 5 Done Done Sent false
5692024-02-03T17:22:45.261Z 21 Acked 1020 Flush 0 Done Done Sent false
5702024-02-03T17:22:45.261Z 22 Acked 1021 Write 5 Done Done Sent false
5712024-02-03T17:22:45.261Z 23 Acked 1022 Write 2 Done Done Sent false
5722024-02-03T17:22:45.261Z 24 Acked 1023 Flush 0 Done Done Sent false
5732024-02-03T17:22:45.261Z 25 Acked 1024 Write 2 Done Done Sent false
5742024-02-03T17:22:45.261Z 26 Acked 1025 Write 8 Done Done Sent false
5752024-02-03T17:22:45.261Z 27 Acked 1026 Read 5 Done Done Sent false
5762024-02-03T17:22:45.261Z 28 Acked 1027 Read 4 Done Done Sent false
5772024-02-03T17:22:45.261Z 29 Acked 1028 Write 9 Done Done Sent false
5782024-02-03T17:22:45.261Z 30 Acked 1029 Read 9 Sent Done Sent false
5792024-02-03T17:22:45.261Z STATES DS:0 DS:1 DS:2 TOTAL
5802024-02-03T17:22:45.261Z New 0 0 0 0
5812024-02-03T17:22:45.261Z Sent 1 0 28 29
5822024-02-03T17:22:45.262Z Done 29 30 2 61
5832024-02-03T17:22:45.262Z Skipped 0 0 0 0
5842024-02-03T17:22:45.262Z Error 0 0 0 0
5852024-02-03T17:22:45.262ZLast Flush: 1023 1023 0
5862024-02-03T17:22:45.262ZDownstairs last five completed:
5872024-02-03T17:22:45.262ZUpstairs last five completed: 30 29 28 27 26
5882024-02-03T17:22:45.262Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:22:44.411983888Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1145}
5892024-02-03T17:22:45.262ZWrote out file "/tmp/test_repair_verify.data"
5902024-02-03T17:22:45.265Z
5912024-02-03T17:22:45.272ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5922024-02-03T17:22:45.297ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5932024-02-03T17:22:45.297Z 0 000-019 4 4 1 4 4 1 F F F <---
5942024-02-03T17:22:45.301Z 1 020-039 1 1 1 1 1 1 F F F
5952024-02-03T17:22:45.301Z 2 040-059 1 1 1 1 1 1 F F F
5962024-02-03T17:22:45.301Z 3 060-079 1 1 1 1 1 1 F F F
5972024-02-03T17:22:45.301Z 4 080-099 1 1 1 1 1 1 F F F
5982024-02-03T17:22:45.301Z 5 100-119 1 1 1 1 1 1 F F F
5992024-02-03T17:22:45.302Z 6 120-139 2 2 2 2 2 2 F F F
6002024-02-03T17:22:45.302Z 7 140-159 1 1 1 1 1 1 F F F
6012024-02-03T17:22:45.302Z 8 160-179 2 2 2 2 2 2 T T F <---
6022024-02-03T17:22:45.302Z 9 180-199 4 4 2 3 3 2 F F T <---
6032024-02-03T17:22:45.302Z 10 200-219 4 4 1 3 3 1 F F T <---
6042024-02-03T17:22:45.302Z 11 220-239 2 2 2 2 2 2 T T F <---
6052024-02-03T17:22:45.302Z 12 240-259 1 1 1 1 1 1 F F F
6062024-02-03T17:22:45.302Z 13 260-279 2 2 2 2 2 2 F F F
6072024-02-03T17:22:45.302Z 14 280-299 2 2 2 2 2 2 F F F
6082024-02-03T17:22:45.302Z 15 300-319 1 1 1 1 1 1 F F F
6092024-02-03T17:22:45.302Z 16 320-339 1 1 1 1 1 1 F F F
6102024-02-03T17:22:45.302Z 17 340-359 1 1 1 1 1 1 F F F
6112024-02-03T17:22:45.302Z 18 360-379 2 2 2 2 2 2 F F F
6122024-02-03T17:22:45.303Z 19 380-399 2 2 2 2 2 2 F F F
6132024-02-03T17:22:45.303Z 20 400-419 4 4 1 6 6 1 F F F <---
6142024-02-03T17:22:45.303Z 21 420-439 1 1 1 1 1 1 F F F
6152024-02-03T17:22:45.303Z 22 440-459 4 4 2 5 5 2 T T F <---
6162024-02-03T17:22:45.303Z 23 460-479 4 4 1 5 5 1 F F F <---
6172024-02-03T17:22:45.303Z 24 480-499 4 4 1 5 5 1 F F F <---
6182024-02-03T17:22:45.303Z 25 500-519 4 4 1 6 6 1 F F F <---
6192024-02-03T17:22:45.303Z 26 520-539 1 1 1 1 1 1 F F F
6202024-02-03T17:22:45.303Z 27 540-559 2 2 2 2 2 2 F F F
6212024-02-03T17:22:45.303Z 28 560-579 2 2 2 2 2 2 F F F
6222024-02-03T17:22:45.303Z 29 580-599 4 4 2 5 5 2 F F F <---
6232024-02-03T17:22:45.303ZMax gen: 4, Max flush: 6
6242024-02-03T17:22:45.303ZError: Difference in extent metadata found!
6252024-02-03T17:22:45.307ZA Difference in extent metadata is expected here
6262024-02-03T17:22:45.307ZOn loop 2
6272024-02-03T17:22:45.307Z
6282024-02-03T17:22:45.307ZVerifying data now
6292024-02-03T17:22:50.313ZNo change after verify
6302024-02-03T17:22:50.314ZLoop: 2 Downstairs dump after verify (and repair):
6312024-02-03T17:22:50.337ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6322024-02-03T17:22:50.337Z 0 000-019 4 4 4 4 4 4 F F F
6332024-02-03T17:22:50.337Z 1 020-039 1 1 1 1 1 1 F F F
6342024-02-03T17:22:50.337Z 2 040-059 1 1 1 1 1 1 F F F
6352024-02-03T17:22:50.337Z 3 060-079 1 1 1 1 1 1 F F F
6362024-02-03T17:22:50.341Z 4 080-099 1 1 1 1 1 1 F F F
6372024-02-03T17:22:50.341Z 5 100-119 1 1 1 1 1 1 F F F
6382024-02-03T17:22:50.341Z 6 120-139 2 2 2 2 2 2 F F F
6392024-02-03T17:22:50.341Z 7 140-159 1 1 1 1 1 1 F F F
6402024-02-03T17:22:50.341Z 8 160-179 5 5 5 7 7 7 F F F
6412024-02-03T17:22:50.341Z 9 180-199 4 4 4 3 3 3 F F F
6422024-02-03T17:22:50.341Z 10 200-219 4 4 4 3 3 3 F F F
6432024-02-03T17:22:50.342Z 11 220-239 5 5 5 7 7 7 F F F
6442024-02-03T17:22:50.342Z 12 240-259 1 1 1 1 1 1 F F F
6452024-02-03T17:22:50.342Z 13 260-279 2 2 2 2 2 2 F F F
6462024-02-03T17:22:50.342Z 14 280-299 2 2 2 2 2 2 F F F
6472024-02-03T17:22:50.342Z 15 300-319 1 1 1 1 1 1 F F F
6482024-02-03T17:22:50.342Z 16 320-339 1 1 1 1 1 1 F F F
6492024-02-03T17:22:50.342Z 17 340-359 1 1 1 1 1 1 F F F
6502024-02-03T17:22:50.342Z 18 360-379 2 2 2 2 2 2 F F F
6512024-02-03T17:22:50.342Z 19 380-399 2 2 2 2 2 2 F F F
6522024-02-03T17:22:50.342Z 20 400-419 4 4 4 6 6 6 F F F
6532024-02-03T17:22:50.342Z 21 420-439 1 1 1 1 1 1 F F F
6542024-02-03T17:22:50.342Z 22 440-459 5 5 5 7 7 7 F F F
6552024-02-03T17:22:50.342Z 23 460-479 4 4 4 5 5 5 F F F
6562024-02-03T17:22:50.342Z 24 480-499 4 4 4 5 5 5 F F F
6572024-02-03T17:22:50.342Z 25 500-519 4 4 4 6 6 6 F F F
6582024-02-03T17:22:50.342Z 26 520-539 1 1 1 1 1 1 F F F
6592024-02-03T17:22:50.342Z 27 540-559 2 2 2 2 2 2 F F F
6602024-02-03T17:22:50.342Z 28 560-579 2 2 2 2 2 2 F F F
6612024-02-03T17:22:50.342Z 29 580-599 4 4 4 5 5 5 F F F
6622024-02-03T17:22:50.342ZMax gen: 5, Max flush: 7
6632024-02-03T17:22:50.346Z
6642024-02-03T17:22:50.346ZBegin loop 3
6652024-02-03T17:22:50.346ZDownstairs to restart: 1
6662024-02-03T17:22:50.376Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.530696418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153}
6672024-02-03T17:22:50.376Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531170787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153}
6682024-02-03T17:22:50.376Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531198202Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153}
6692024-02-03T17:22:50.380Z{"msg":"Crucible 8e60a741-4c41-4785-b6dd-33056f564108 has session id: 955daf38-c680-4017-a298-94664ce313a4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531225678Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6702024-02-03T17:22:50.380Z{"msg":"Upstairs opts: Upstairs UUID: 8e60a741-4c41-4785-b6dd-33056f564108, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531312082Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6712024-02-03T17:22:50.380Z{"msg":"Crucible runtime is spawned
6722024-02-03T17:22:50.380ZCrucible stats registered with UUID: 8e60a741-4c41-4785-b6dd-33056f564108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531451959Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6732024-02-03T17:22:50.380Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531564501Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153}
6742024-02-03T17:22:50.380Z{"msg":"8e60a741-4c41-4785-b6dd-33056f564108 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531731058Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6752024-02-03T17:22:50.380Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531809144Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6762024-02-03T17:22:50.380Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531857567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6772024-02-03T17:22:50.380Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531900369Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6782024-02-03T17:22:50.380Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531943812Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6792024-02-03T17:22:50.380Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.531981924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6802024-02-03T17:22:50.381Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.532102561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6812024-02-03T17:22:50.381Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.532147136Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6822024-02-03T17:22:50.381Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.532192329Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6832024-02-03T17:22:50.381Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.532427781Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6842024-02-03T17:22:50.381Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.532540105Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6852024-02-03T17:22:50.381Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.532584212Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6862024-02-03T17:22:50.381Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.53264178Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6872024-02-03T17:22:50.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.533778548Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6882024-02-03T17:22:50.381Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.533860542Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6892024-02-03T17:22:50.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.533885542Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6902024-02-03T17:22:50.381Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.533910738Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6912024-02-03T17:22:50.381Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.534762646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6922024-02-03T17:22:50.381Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.534956373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6932024-02-03T17:22:50.381Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.535186003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6942024-02-03T17:22:50.381Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.535228614Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6952024-02-03T17:22:50.385Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.535507572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6962024-02-03T17:22:50.385Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.535588144Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6972024-02-03T17:22:50.385Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.53563564Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6982024-02-03T17:22:50.386Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.535900084Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
6992024-02-03T17:22:50.386Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.535970592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7002024-02-03T17:22:50.386Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.536003533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7012024-02-03T17:22:50.386Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.53626086Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7022024-02-03T17:22:50.386Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.536963159Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7032024-02-03T17:22:50.386Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537027112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7042024-02-03T17:22:50.386Z{"msg":"[0]R flush_numbers[0..12]: [4, 1, 1, 1, 1, 1, 2, 1, 7, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537166637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7052024-02-03T17:22:50.386Z{"msg":"[0]R generation[0..12]: [4, 1, 1, 1, 1, 1, 2, 1, 5, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537229982Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7062024-02-03T17:22:50.386Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537259229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7072024-02-03T17:22:50.386Z{"msg":"[1]R flush_numbers[0..12]: [4, 1, 1, 1, 1, 1, 2, 1, 7, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537302227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7082024-02-03T17:22:50.386Z{"msg":"[1]R generation[0..12]: [4, 1, 1, 1, 1, 1, 2, 1, 5, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537327369Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7092024-02-03T17:22:50.386Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537370434Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7102024-02-03T17:22:50.386Z{"msg":"[2]R flush_numbers[0..12]: [4, 1, 1, 1, 1, 1, 2, 1, 7, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
7112024-02-03T17:22:50.386Z2024-02-03T17:22:49.537433623Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7122024-02-03T17:22:50.386Z{"msg":"[2]R generation[0..12]: [4, 1, 1, 1, 1, 1, 2, 1, 5, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.53748216Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7132024-02-03T17:22:50.386Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537541928Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7142024-02-03T17:22:50.386Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537585279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7152024-02-03T17:22:50.386Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537645213Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7162024-02-03T17:22:50.386Z{"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537687994Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7172024-02-03T17:22:50.386Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537740957Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7182024-02-03T17:22:50.386Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537784112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7192024-02-03T17:22:50.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537836444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7202024-02-03T17:22:50.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537879791Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7212024-02-03T17:22:50.386Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537934085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7222024-02-03T17:22:50.387Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.537976561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7232024-02-03T17:22:50.387Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.53801534Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7242024-02-03T17:22:50.387Z{"msg":"8e60a741-4c41-4785-b6dd-33056f564108 is now active with session: 955daf38-c680-4017-a298-94664ce313a4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.53808183Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7252024-02-03T17:22:50.387Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.538121998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153,"session_id":"955daf38-c680-4017-a298-94664ce313a4"}
7262024-02-03T17:22:50.387Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:49.538174757Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153}
7272024-02-03T17:22:50.387ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7282024-02-03T17:22:50.387ZLoading write count information from file "/tmp/test_repair_verify.data"
7292024-02-03T17:22:50.387ZRun Repair workload
7302024-02-03T17:22:50.387Z01/30 Write block 260 len 36864 data: 2 2 2 2 2 2 2 2 2
7312024-02-03T17:22:50.390Z02/30 Read block 193 len 40960
7322024-02-03T17:22:50.391Z03/30 Read block 251 len 4096
7332024-02-03T17:22:50.391Z04/30 Read block 344 len 8192
7342024-02-03T17:22:50.397Z05/30 Read block 436 len 12288
7352024-02-03T17:22:50.397Z06/30 Read block 70 len 40960
7362024-02-03T17:22:50.397Z07/30 Write block 292 len 8192 data: 2 2
7372024-02-03T17:22:50.400Z08/30 Write block 44 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7382024-02-03T17:22:50.400Z09/30 Write block 154 len 36864 data: 2 2 2 2 2 2 2 2 3
7392024-02-03T17:22:50.400Z10/30 Read block 156 len 12288
7402024-02-03T17:22:50.400Z11/30 Read block 92 len 12288
7412024-02-03T17:22:50.403Z12/30 Flush
7422024-02-03T17:22:50.407Z13/30 Write block 376 len 12288 data: 2 2 2
7432024-02-03T17:22:50.407Z14/30 Flush
7442024-02-03T17:22:50.410Z15/30 Write block 457 len 12288 data: 3 3 3
7452024-02-03T17:22:50.410Z16/30 Write block 135 len 32768 data: 3 3 3 3 3 2 2 2
7462024-02-03T17:22:50.410Z17/30 Read block 291 len 32768
7472024-02-03T17:22:50.413Z18/30 Write block 465 len 4096 data: 2
7482024-02-03T17:22:50.413Z19/30 Write block 4 len 24576 data: 2 2 2 2 3 2
7492024-02-03T17:22:50.416Z20/30 Flush
7502024-02-03T17:22:50.419Z21/30 Write block 552 len 8192 data: 3 3
7512024-02-03T17:22:50.419Z22/30 Flush
7522024-02-03T17:22:50.423Z23/30 Write block 440 len 12288 data: 2 2 2
7532024-02-03T17:22:50.423Z24/30 Read block 532 len 16384
7542024-02-03T17:22:50.426Z25/30 Write block 468 len 8192 data: 2 2
7552024-02-03T17:22:50.426Z26/30 Write block 579 len 20480 data: 2 2 2 2 2
7562024-02-03T17:22:50.426Z27/30 Read block 368 len 8192
7572024-02-03T17:22:50.426Z28/30 Read block 234 len 8192
7582024-02-03T17:22:50.430Z29/30 Write block 444 len 36864 data: 3 2 2 3 4 4 4 4 4
7592024-02-03T17:22:50.430Z30/30 Write block 49 len 20480 data: 3 3 3 3 3
7602024-02-03T17:22:50.430Z----------------------------------------------------------------
7612024-02-03T17:22:50.430Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7622024-02-03T17:22:50.430ZStates: Active Active Active
7632024-02-03T17:22:50.430ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7642024-02-03T17:22:50.430Z 1 Acked 1000 Write 9 Done Done Done false
7652024-02-03T17:22:50.430Z 2 Acked 1001 Read 10 Done Done Done false
7662024-02-03T17:22:50.430Z 3 Acked 1002 Read 1 Done Sent Done false
7672024-02-03T17:22:50.430Z 4 Acked 1003 Read 2 Done Sent Done false
7682024-02-03T17:22:50.430Z 5 Acked 1004 Read 3 Done Sent Done false
7692024-02-03T17:22:50.431Z 6 Acked 1005 Read 10 Done Sent Done false
7702024-02-03T17:22:50.431Z 7 Acked 1006 Write 2 Done Sent Done false
7712024-02-03T17:22:50.431Z 8 Acked 1007 Write 10 Done Sent Done false
7722024-02-03T17:22:50.431Z 9 Acked 1008 Write 9 Done Sent Done false
7732024-02-03T17:22:50.431Z 10 Acked 1009 Read 3 Done Sent Done false
7742024-02-03T17:22:50.431Z 11 Acked 1010 Read 3 Done Sent Done false
7752024-02-03T17:22:50.431Z 12 Acked 1011 Flush 0 Done Sent Done false
7762024-02-03T17:22:50.431Z 13 Acked 1012 Write 3 Done Sent Done false
7772024-02-03T17:22:50.431Z 14 Acked 1013 Flush 0 Done Sent Done false
7782024-02-03T17:22:50.431Z 15 Acked 1014 Write 3 Done Sent Done false
7792024-02-03T17:22:50.431Z 16 Acked 1015 Write 8 Done Sent Done false
7802024-02-03T17:22:50.431Z 17 Acked 1016 Read 8 Done Sent Done false
7812024-02-03T17:22:50.431Z 18 Acked 1017 Write 1 Done Sent Done false
7822024-02-03T17:22:50.431Z 19 Acked 1018 Write 6 Done Sent Done false
7832024-02-03T17:22:50.431Z 20 Acked 1019 Flush 0 Done Sent Done false
7842024-02-03T17:22:50.431Z 21 Acked 1020 Write 2 Done Sent Done false
7852024-02-03T17:22:50.431Z 22 Acked 1021 Flush 0 Done Sent Done false
7862024-02-03T17:22:50.431Z 23 Acked 1022 Write 3 Done Sent Done false
7872024-02-03T17:22:50.431Z 24 Acked 1023 Read 4 Done Sent Done false
7882024-02-03T17:22:50.431Z 25 Acked 1024 Write 2 Done Sent Done false
7892024-02-03T17:22:50.431Z 26 Acked 1025 Write 5 Done Sent Done false
7902024-02-03T17:22:50.431Z 27 Acked 1026 Read 2 Done Sent Done false
7912024-02-03T17:22:50.431Z 28 Acked 1027 Read 2 Done Sent Done false
7922024-02-03T17:22:50.431Z 29 Acked 1028 Write 9 Sent Sent Sent false
7932024-02-03T17:22:50.432Z 30 Acked 1029 Write 5 Sent Sent Sent false
7942024-02-03T17:22:50.432Z STATES DS:0 DS:1 DS:2 TOTAL
7952024-02-03T17:22:50.432Z New 0 0 0 0
7962024-02-03T17:22:50.432Z Sent 2 28 2 32
7972024-02-03T17:22:50.432Z Done 28 2 28 58
7982024-02-03T17:22:50.432Z Skipped 0 0 0 0
7992024-02-03T17:22:50.432Z Error 0 0 0 0
8002024-02-03T17:22:50.432ZLast Flush: 1021 0 1021
8012024-02-03T17:22:50.432ZDownstairs last five completed:
8022024-02-03T17:22:50.432ZUpstairs last five completed: 30 29 28 27 26
8032024-02-03T17:22:50.435Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:22:49.585824694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1153}
8042024-02-03T17:22:50.435ZWrote out file "/tmp/test_repair_verify.data"
8052024-02-03T17:22:50.439Z
8062024-02-03T17:22:50.445ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8072024-02-03T17:22:50.472ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8082024-02-03T17:22:50.472Z 0 000-019 6 4 6 10 4 10 F F F <---
8092024-02-03T17:22:50.476Z 1 020-039 1 1 1 1 1 1 F F F
8102024-02-03T17:22:50.476Z 2 040-059 6 1 6 8 1 8 T F T <---
8112024-02-03T17:22:50.476Z 3 060-079 1 1 1 1 1 1 F F F
8122024-02-03T17:22:50.476Z 4 080-099 1 1 1 1 1 1 F F F
8132024-02-03T17:22:50.476Z 5 100-119 1 1 1 1 1 1 F F F
8142024-02-03T17:22:50.476Z 6 120-139 6 2 6 10 2 10 F F F <---
8152024-02-03T17:22:50.476Z 7 140-159 6 1 6 10 1 10 F F F <---
8162024-02-03T17:22:50.476Z 8 160-179 6 5 6 8 7 8 F F F <---
8172024-02-03T17:22:50.477Z 9 180-199 4 4 4 3 3 3 F F F
8182024-02-03T17:22:50.477Z 10 200-219 4 4 4 3 3 3 F F F
8192024-02-03T17:22:50.477Z 11 220-239 5 5 5 7 7 7 F F F
8202024-02-03T17:22:50.477Z 12 240-259 1 1 1 1 1 1 F F F
8212024-02-03T17:22:50.477Z 13 260-279 6 2 6 8 2 8 F T F <---
8222024-02-03T17:22:50.477Z 14 280-299 6 2 6 8 2 8 F F F <---
8232024-02-03T17:22:50.477Z 15 300-319 1 1 1 1 1 1 F F F
8242024-02-03T17:22:50.477Z 16 320-339 1 1 1 1 1 1 F F F
8252024-02-03T17:22:50.477Z 17 340-359 1 1 1 1 1 1 F F F
8262024-02-03T17:22:50.477Z 18 360-379 6 2 6 9 2 9 F F F <---
8272024-02-03T17:22:50.477Z 19 380-399 2 2 2 2 2 2 F F F
8282024-02-03T17:22:50.477Z 20 400-419 4 4 4 6 6 6 F F F
8292024-02-03T17:22:50.477Z 21 420-439 1 1 1 1 1 1 F F F
8302024-02-03T17:22:50.477Z 22 440-459 6 5 6 10 7 10 T F T <---
8312024-02-03T17:22:50.477Z 23 460-479 6 4 6 10 5 10 T F T <---
8322024-02-03T17:22:50.477Z 24 480-499 4 4 4 5 5 5 F F F
8332024-02-03T17:22:50.477Z 25 500-519 4 4 4 6 6 6 F F F
8342024-02-03T17:22:50.477Z 26 520-539 1 1 1 1 1 1 F F F
8352024-02-03T17:22:50.477Z 27 540-559 6 2 6 11 2 11 F F F <---
8362024-02-03T17:22:50.477Z 28 560-579 2 2 2 2 2 2 T F T <---
8372024-02-03T17:22:50.477Z 29 580-599 4 4 4 5 5 5 T F T <---
8382024-02-03T17:22:50.477ZMax gen: 6, Max flush: 11
8392024-02-03T17:22:50.477ZError: Difference in extent metadata found!
8402024-02-03T17:22:50.484ZA Difference in extent metadata is expected here
8412024-02-03T17:22:50.484ZOn loop 3
8422024-02-03T17:22:50.484Z
8432024-02-03T17:22:50.487ZVerifying data now
8442024-02-03T17:22:55.612ZNo change after verify
8452024-02-03T17:22:55.612ZLoop: 3 Downstairs dump after verify (and repair):
8462024-02-03T17:22:55.636ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8472024-02-03T17:22:55.636Z 0 000-019 6 6 6 10 10 10 F F F
8482024-02-03T17:22:55.640Z 1 020-039 1 1 1 1 1 1 F F F
8492024-02-03T17:22:55.640Z 2 040-059 7 7 7 12 12 12 F F F
8502024-02-03T17:22:55.640Z 3 060-079 1 1 1 1 1 1 F F F
8512024-02-03T17:22:55.640Z 4 080-099 1 1 1 1 1 1 F F F
8522024-02-03T17:22:55.640Z 5 100-119 1 1 1 1 1 1 F F F
8532024-02-03T17:22:55.640Z 6 120-139 6 6 6 10 10 10 F F F
8542024-02-03T17:22:55.640Z 7 140-159 6 6 6 10 10 10 F F F
8552024-02-03T17:22:55.640Z 8 160-179 6 6 6 8 8 8 F F F
8562024-02-03T17:22:55.640Z 9 180-199 4 4 4 3 3 3 F F F
8572024-02-03T17:22:55.640Z 10 200-219 4 4 4 3 3 3 F F F
8582024-02-03T17:22:55.640Z 11 220-239 5 5 5 7 7 7 F F F
8592024-02-03T17:22:55.640Z 12 240-259 1 1 1 1 1 1 F F F
8602024-02-03T17:22:55.640Z 13 260-279 6 6 6 8 8 8 F F F
8612024-02-03T17:22:55.640Z 14 280-299 6 6 6 8 8 8 F F F
8622024-02-03T17:22:55.640Z 15 300-319 1 1 1 1 1 1 F F F
8632024-02-03T17:22:55.640Z 16 320-339 1 1 1 1 1 1 F F F
8642024-02-03T17:22:55.640Z 17 340-359 1 1 1 1 1 1 F F F
8652024-02-03T17:22:55.640Z 18 360-379 6 6 6 9 9 9 F F F
8662024-02-03T17:22:55.640Z 19 380-399 2 2 2 2 2 2 F F F
8672024-02-03T17:22:55.640Z 20 400-419 4 4 4 6 6 6 F F F
8682024-02-03T17:22:55.640Z 21 420-439 1 1 1 1 1 1 F F F
8692024-02-03T17:22:55.641Z 22 440-459 7 7 7 12 12 12 F F F
8702024-02-03T17:22:55.641Z 23 460-479 7 7 7 12 12 12 F F F
8712024-02-03T17:22:55.641Z 24 480-499 4 4 4 5 5 5 F F F
8722024-02-03T17:22:55.641Z 25 500-519 4 4 4 6 6 6 F F F
8732024-02-03T17:22:55.641Z 26 520-539 1 1 1 1 1 1 F F F
8742024-02-03T17:22:55.641Z 27 540-559 6 6 6 11 11 11 F F F
8752024-02-03T17:22:55.641Z 28 560-579 7 7 7 12 12 12 F F F
8762024-02-03T17:22:55.641Z 29 580-599 7 7 7 12 12 12 F F F
8772024-02-03T17:22:55.641ZMax gen: 7, Max flush: 12
8782024-02-03T17:22:55.644Z
8792024-02-03T17:22:55.644ZBegin loop 4
8802024-02-03T17:22:55.644ZDownstairs to restart: 2
8812024-02-03T17:22:55.675Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.830349144Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161}
8822024-02-03T17:22:55.675Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.830905612Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161}
8832024-02-03T17:22:55.676Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.830938066Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161}
8842024-02-03T17:22:55.681Z{"msg":"Crucible 5850db7f-1a6e-43e9-8a45-58bb03ee1b6e has session id: b1f2f1e7-6d9b-4744-8175-32c714ed33b8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.830976599Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8852024-02-03T17:22:55.681Z{"msg":"Upstairs opts: Upstairs UUID: 5850db7f-1a6e-43e9-8a45-58bb03ee1b6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
8862024-02-03T17:22:55.681Z,"time":"2024-02-03T17:22:54.831064597Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8872024-02-03T17:22:55.681Z{"msg":"Crucible stats registered with UUID: 5850db7f-1a6e-43e9-8a45-58bb03ee1b6e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831285461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8882024-02-03T17:22:55.681Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831352848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161}
8892024-02-03T17:22:55.681Z{"msg":"5850db7f-1a6e-43e9-8a45-58bb03ee1b6e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831501825Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8902024-02-03T17:22:55.681Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831560547Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8912024-02-03T17:22:55.682Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831587197Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8922024-02-03T17:22:55.682Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831618277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8932024-02-03T17:22:55.682Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831685735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8942024-02-03T17:22:55.682Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831760923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8952024-02-03T17:22:55.682Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831807495Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8962024-02-03T17:22:55.682Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.831877591Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8972024-02-03T17:22:55.682Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.832168816Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8982024-02-03T17:22:55.682Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.832243829Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
8992024-02-03T17:22:55.682Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.832289971Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9002024-02-03T17:22:55.682Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.832361937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9012024-02-03T17:22:55.682Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.832391383Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9022024-02-03T17:22:55.682Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.833346003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9032024-02-03T17:22:55.683Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.833402405Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9042024-02-03T17:22:55.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.833502715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9052024-02-03T17:22:55.683Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.83357465Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9062024-02-03T17:22:55.683Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.834253248Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9072024-02-03T17:22:55.683Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.834339736Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9082024-02-03T17:22:55.683Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.83438767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9092024-02-03T17:22:55.683Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.834421712Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9102024-02-03T17:22:55.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.835166471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9112024-02-03T17:22:55.683Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.835248442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9122024-02-03T17:22:55.683Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.835276863Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9132024-02-03T17:22:55.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.835310611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9142024-02-03T17:22:55.683Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.835373551Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9152024-02-03T17:22:55.683Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.835398874Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9162024-02-03T17:22:55.684Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.83542124Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9172024-02-03T17:22:55.688Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836227747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9182024-02-03T17:22:55.688Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836286851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9192024-02-03T17:22:55.688Z{"msg":"[0]R flush_numbers[0..12]: [10, 1, 12, 1, 1, 1, 10, 10, 8, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836332263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9202024-02-03T17:22:55.688Z{"msg":"[0]R generation[0..12]: [6, 1, 7, 1, 1, 1, 6, 6, 6, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836374066Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9212024-02-03T17:22:55.689Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836433228Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9222024-02-03T17:22:55.689Z{"msg":"[1]R flush_numbers[0..12]: [10, 1, 12, 1, 1, 1, 10, 10, 8, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836502916Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9232024-02-03T17:22:55.689Z{"msg":"[1]R generation[0..12]: [6, 1, 7, 1, 1, 1, 6, 6, 6, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836543524Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9242024-02-03T17:22:55.689Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
9252024-02-03T17:22:55.689Z,"time":"2024-02-03T17:22:54.836615916Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9262024-02-03T17:22:55.689Z{"msg":"[2]R flush_numbers[0..12]: [10, 1, 12, 1, 1, 1, 10, 10, 8, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836655241Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9272024-02-03T17:22:55.689Z{"msg":"[2]R generation[0..12]: [6, 1, 7, 1, 1, 1, 6, 6, 6, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836719249Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9282024-02-03T17:22:55.689Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836754727Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9292024-02-03T17:22:55.689Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836840859Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9302024-02-03T17:22:55.689Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836885753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9312024-02-03T17:22:55.689Z{"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836943822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9322024-02-03T17:22:55.689Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.836986003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9332024-02-03T17:22:55.689Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.837046389Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9342024-02-03T17:22:55.689Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.837086106Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9352024-02-03T17:22:55.689Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.837146117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9362024-02-03T17:22:55.690Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.837188991Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9372024-02-03T17:22:55.690Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.837251436Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9382024-02-03T17:22:55.690Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.837291848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9392024-02-03T17:22:55.690Zb1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9402024-02-03T17:22:55.690Z{"msg":"5850db7f-1a6e-43e9-8a45-58bb03ee1b6e is now active with session: b1f2f1e7-6d9b-4744-8175-32c714ed33b8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.837404695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9412024-02-03T17:22:55.690Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.83745876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161,"session_id":"b1f2f1e7-6d9b-4744-8175-32c714ed33b8"}
9422024-02-03T17:22:55.690Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:22:54.837499114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161}
9432024-02-03T17:22:55.690ZLoading write count information from file "/tmp/test_repair_verify.data"
9442024-02-03T17:22:55.690ZRun Repair workload
9452024-02-03T17:22:55.690Z01/30 Write block 393 len 32768 data: 3 3 3 3 2 2 2 2
9462024-02-03T17:22:55.690Z02/30 Write block 570 len 12288 data: 2 2 2
9472024-02-03T17:22:55.690Z03/30 Read block 478 len 16384
9482024-02-03T17:22:55.696Z04/30 Read block 82 len 28672
9492024-02-03T17:22:55.696Z05/30 Read block 425 len 8192
9502024-02-03T17:22:55.696Z06/30 Read block 108 len 36864
9512024-02-03T17:22:55.696Z07/30 Read block 334 len 40960
9522024-02-03T17:22:55.700Z08/30 Write block 564 len 20480 data: 2 2 2 2 2
9532024-02-03T17:22:55.700Z09/30 Read block 582 len 40960
9542024-02-03T17:22:55.700Z10/30 Read block 346 len 36864
9552024-02-03T17:22:55.700Z11/30 Write block 529 len 8192 data: 2 2
9562024-02-03T17:22:55.700Z12/30 Read block 129 len 4096
9572024-02-03T17:22:55.703Z13/30 Flush
9582024-02-03T17:22:55.706Z14/30 Write block 577 len 40960 data: 2 2 3 3 3 3 3 2 2 2
9592024-02-03T17:22:55.707Z15/30 Flush
9602024-02-03T17:22:55.710Z16/30 Write block 526 len 8192 data: 2 2
9612024-02-03T17:22:55.710Z17/30 Flush
9622024-02-03T17:22:55.714Z18/30 Write block 383 len 28672 data: 2 2 2 2 2 2 2
9632024-02-03T17:22:55.714Z19/30 Write block 463 len 40960 data: 2 2 3 2 2 3 3 2 2 2
9642024-02-03T17:22:55.717Z20/30 Read block 260 len 16384
9652024-02-03T17:22:55.717Z21/30 Read block 512 len 24576
9662024-02-03T17:22:55.720Z22/30 Read block 350 len 4096
9672024-02-03T17:22:55.720Z23/30 Read block 114 len 36864
9682024-02-03T17:22:55.721Z24/30 Read block 518 len 12288
9692024-02-03T17:22:55.724Z25/30 Read block 278 len 24576
9702024-02-03T17:22:55.724Z26/30 Write block 430 len 12288 data: 2 2 2
9712024-02-03T17:22:55.727Z27/30 Write block 21 len 20480 data: 2 2 2 2 2
9722024-02-03T17:22:55.727Z28/30 Read block 99 len 8192
9732024-02-03T17:22:55.728Z29/30 Write block 367 len 32768 data: 3 3 2 2 2 2 2 2
9742024-02-03T17:22:55.732Z30/30 Write block 556 len 16384 data: 3 2 2 3
9752024-02-03T17:22:55.732Z----------------------------------------------------------------
9762024-02-03T17:22:55.732Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9772024-02-03T17:22:55.732ZStates: Active Active Active
9782024-02-03T17:22:55.732ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9792024-02-03T17:22:55.732Z 1 Acked 1000 Write 8 Done Done Done false
9802024-02-03T17:22:55.732Z 2 Acked 1001 Write 3 Done Done Sent false
9812024-02-03T17:22:55.733Z 3 Acked 1002 Read 4 Done Done Sent false
9822024-02-03T17:22:55.733Z 4 Acked 1003 Read 7 Done Done Sent false
9832024-02-03T17:22:55.733Z 5 Acked 1004 Read 2 Done Done Sent false
9842024-02-03T17:22:55.733Z 6 Acked 1005 Read 9 Done Done Sent false
9852024-02-03T17:22:55.733Z 7 Acked 1006 Read 10 Done Done Sent false
9862024-02-03T17:22:55.733Z 8 Acked 1007 Write 5 Done Done Sent false
9872024-02-03T17:22:55.733Z 9 Acked 1008 Read 10 Done Done Sent false
9882024-02-03T17:22:55.733Z 10 Acked 1009 Read 9 Done Done Sent false
9892024-02-03T17:22:55.733Z 11 Acked 1010 Write 2 Done Done Sent false
9902024-02-03T17:22:55.733Z 12 Acked 1011 Read 1 Done Done Sent false
9912024-02-03T17:22:55.733Z 13 Acked 1012 Flush 0 Done Done Sent false
9922024-02-03T17:22:55.733Z 14 Acked 1013 Write 10 Done Done Sent false
9932024-02-03T17:22:55.733Z 15 Acked 1014 Flush 0 Done Done Sent false
9942024-02-03T17:22:55.733Z 16 Acked 1015 Write 2 Done Done Sent false
9952024-02-03T17:22:55.733Z 17 Acked 1016 Flush 0 Done Done Sent false
9962024-02-03T17:22:55.734Z 18 Acked 1017 Write 7 Done Done Sent false
9972024-02-03T17:22:55.734Z 19 Acked 1018 Write 10 Done Done Sent false
9982024-02-03T17:22:55.734Z 20 Acked 1019 Read 4 Done Done Sent false
9992024-02-03T17:22:55.734Z 21 Acked 1020 Read 6 Done Done Sent false
10002024-02-03T17:22:55.734Z 22 Acked 1021 Read 1 Done Done Sent false
10012024-02-03T17:22:55.734Z 23 Acked 1022 Read 9 Done Done Sent false
10022024-02-03T17:22:55.734Z 24 Acked 1023 Read 3 Done Done Sent false
10032024-02-03T17:22:55.734Z 25 Acked 1024 Read 6 Done Done Sent false
10042024-02-03T17:22:55.734Z 26 Acked 1025 Write 3 Done Done Sent false
10052024-02-03T17:22:55.734Z 27 Acked 1026 Write 5 Done Done Sent false
10062024-02-03T17:22:55.734Z 28 Acked 1027 Read 2 Done Done Sent false
10072024-02-03T17:22:55.734Z 29 Acked 1028 Write 8 Sent Sent Sent false
10082024-02-03T17:22:55.734Z 30 Acked 1029 Write 4 Sent Sent Sent false
10092024-02-03T17:22:55.734Z STATES DS:0 DS:1 DS:2 TOTAL
10102024-02-03T17:22:55.735Z New 0 0 0 0
10112024-02-03T17:22:55.735Z Sent 2 2 29 33
10122024-02-03T17:22:55.735Z Done 28 28 1 57
10132024-02-03T17:22:55.735Z Skipped 0 0 0 0
10142024-02-03T17:22:55.735Z Error 0 0 0 0
10152024-02-03T17:22:55.735ZLast Flush: 1016 1016 0
10162024-02-03T17:22:55.735ZDownstairs last five completed:
10172024-02-03T17:22:55.735ZUpstairs last five completed: 30 29 28 27 26
10182024-02-03T17:22:55.735Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:22:54.885262577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1161}
10192024-02-03T17:22:55.735ZWrote out file "/tmp/test_repair_verify.data"
10202024-02-03T17:22:55.740Z
10212024-02-03T17:22:55.745ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10222024-02-03T17:22:55.774ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10232024-02-03T17:22:55.775Z 0 000-019 6 6 6 10 10 10 F F F
10242024-02-03T17:22:55.775Z 1 020-039 1 1 1 1 1 1 T T F <---
10252024-02-03T17:22:55.775Z 2 040-059 7 7 7 12 12 12 F F F
10262024-02-03T17:22:55.775Z 3 060-079 1 1 1 1 1 1 F F F
10272024-02-03T17:22:55.775Z 4 080-099 1 1 1 1 1 1 F F F
10282024-02-03T17:22:55.775Z 5 100-119 1 1 1 1 1 1 F F F
10292024-02-03T17:22:55.775Z 6 120-139 6 6 6 10 10 10 F F F
10302024-02-03T17:22:55.775Z 7 140-159 6 6 6 10 10 10 F F F
10312024-02-03T17:22:55.775Z 8 160-179 6 6 6 8 8 8 F F F
10322024-02-03T17:22:55.775Z 9 180-199 4 4 4 3 3 3 F F F
10332024-02-03T17:22:55.775Z 10 200-219 4 4 4 3 3 3 F F F
10342024-02-03T17:22:55.775Z 11 220-239 5 5 5 7 7 7 F F F
10352024-02-03T17:22:55.775Z 12 240-259 1 1 1 1 1 1 F F F
10362024-02-03T17:22:55.779Z 13 260-279 6 6 6 8 8 8 F F F
10372024-02-03T17:22:55.780Z 14 280-299 6 6 6 8 8 8 F F F
10382024-02-03T17:22:55.780Z 15 300-319 1 1 1 1 1 1 F F F
10392024-02-03T17:22:55.780Z 16 320-339 1 1 1 1 1 1 F F F
10402024-02-03T17:22:55.780Z 17 340-359 1 1 1 1 1 1 F F F
10412024-02-03T17:22:55.780Z 18 360-379 6 6 6 9 9 9 T T F <---
10422024-02-03T17:22:55.780Z 19 380-399 8 8 2 13 13 2 T T T <---
10432024-02-03T17:22:55.780Z 20 400-419 8 8 4 13 13 6 F F T <---
10442024-02-03T17:22:55.780Z 21 420-439 1 1 1 1 1 1 T T F <---
10452024-02-03T17:22:55.780Z 22 440-459 7 7 7 12 12 12 F F F
10462024-02-03T17:22:55.780Z 23 460-479 7 7 7 12 12 12 T T F <---
10472024-02-03T17:22:55.780Z 24 480-499 4 4 4 5 5 5 F F F
10482024-02-03T17:22:55.780Z 25 500-519 4 4 4 6 6 6 F F F
10492024-02-03T17:22:55.780Z 26 520-539 8 8 1 15 15 1 F F F <---
10502024-02-03T17:22:55.780Z 27 540-559 6 6 6 11 11 11 T T F <---
10512024-02-03T17:22:55.781Z 28 560-579 8 8 7 14 14 12 F F F <---
10522024-02-03T17:22:55.781Z 29 580-599 8 8 7 14 14 12 F F F <---
10532024-02-03T17:22:55.781ZMax gen: 8, Max flush: 15
10542024-02-03T17:22:55.781ZError: Difference in extent metadata found!
10552024-02-03T17:22:55.781ZA Difference in extent metadata is expected here
10562024-02-03T17:22:55.781ZOn loop 4
10572024-02-03T17:22:55.781Z
10582024-02-03T17:22:55.786ZVerifying data now
10592024-02-03T17:23:01.723ZNo change after verify
10602024-02-03T17:23:01.723ZLoop: 4 Downstairs dump after verify (and repair):
10612024-02-03T17:23:01.748ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10622024-02-03T17:23:01.748Z 0 000-019 6 6 6 10 10 10 F F F
10632024-02-03T17:23:01.748Z 1 020-039 9 9 9 16 16 16 F F F
10642024-02-03T17:23:01.748Z 2 040-059 7 7 7 12 12 12 F F F
10652024-02-03T17:23:01.748Z 3 060-079 1 1 1 1 1 1 F F F
10662024-02-03T17:23:01.748Z 4 080-099 1 1 1 1 1 1 F F F
10672024-02-03T17:23:01.752Z 5 100-119 1 1 1 1 1 1 F F F
10682024-02-03T17:23:01.752Z 6 120-139 6 6 6 10 10 10 F F F
10692024-02-03T17:23:01.752Z 7 140-159 6 6 6 10 10 10 F F F
10702024-02-03T17:23:01.752Z 8 160-179 6 6 6 8 8 8 F F F
10712024-02-03T17:23:01.752Z 9 180-199 4 4 4 3 3 3 F F F
10722024-02-03T17:23:01.752Z 10 200-219 4 4 4 3 3 3 F F F
10732024-02-03T17:23:01.752Z 11 220-239 5 5 5 7 7 7 F F F
10742024-02-03T17:23:01.753Z 12 240-259 1 1 1 1 1 1 F F F
10752024-02-03T17:23:01.753Z 13 260-279 6 6 6 8 8 8 F F F
10762024-02-03T17:23:01.753Z 14 280-299 6 6 6 8 8 8 F F F
10772024-02-03T17:23:01.753Z 15 300-319 1 1 1 1 1 1 F F F
10782024-02-03T17:23:01.753Z 16 320-339 1 1 1 1 1 1 F F F
10792024-02-03T17:23:01.753Z 17 340-359 1 1 1 1 1 1 F F F
10802024-02-03T17:23:01.753Z 18 360-379 9 9 9 16 16 16 F F F
10812024-02-03T17:23:01.753Z 19 380-399 9 9 9 16 16 16 F F F
10822024-02-03T17:23:01.753Z 20 400-419 8 8 8 13 13 13 F F F
10832024-02-03T17:23:01.753Z 21 420-439 9 9 9 16 16 16 F F F
10842024-02-03T17:23:01.753Z 22 440-459 7 7 7 12 12 12 F F F
10852024-02-03T17:23:01.753Z 23 460-479 9 9 9 16 16 16 F F F
10862024-02-03T17:23:01.753Z 24 480-499 4 4 4 5 5 5 F F F
10872024-02-03T17:23:01.753Z 25 500-519 4 4 4 6 6 6 F F F
10882024-02-03T17:23:01.753Z 26 520-539 8 8 8 15 15 15 F F F
10892024-02-03T17:23:01.753Z 27 540-559 9 9 9 16 16 16 F F F
10902024-02-03T17:23:01.753Z 28 560-579 8 8 8 14 14 14 F F F
10912024-02-03T17:23:01.753Z 29 580-599 8 8 8 14 14 14 F F F
10922024-02-03T17:23:01.753ZMax gen: 9, Max flush: 16
10932024-02-03T17:23:01.757Z
10942024-02-03T17:23:01.757ZBegin loop 5
10952024-02-03T17:23:01.757ZDownstairs to restart: 1
10962024-02-03T17:23:01.788Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.943955477Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169}
10972024-02-03T17:23:01.788Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.944470191Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169}
10982024-02-03T17:23:01.789Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.9445176Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169}
10992024-02-03T17:23:01.794ZCrucible runtime is spawned
11002024-02-03T17:23:01.794Z{"msg":"Crucible ed624788-6c86-4739-b213-1f4396bdd2a4 has session id: a1b3ec91-f393-4cce-8c07-da724301261a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.944741261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11012024-02-03T17:23:01.794Z{"msg":"Upstairs opts: Upstairs UUID: ed624788-6c86-4739-b213-1f4396bdd2a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.944896166Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11022024-02-03T17:23:01.794Z{"msg":"Crucible stats registered with UUID: ed624788-6c86-4739-b213-1f4396bdd2a4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.944991475Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11032024-02-03T17:23:01.794Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.945041084Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169}
11042024-02-03T17:23:01.794Z{"msg":"ed624788-6c86-4739-b213-1f4396bdd2a4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.945136804Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11052024-02-03T17:23:01.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.945181561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11062024-02-03T17:23:01.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.945265322Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11072024-02-03T17:23:01.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.94536828Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11082024-02-03T17:23:01.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.945425658Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11092024-02-03T17:23:01.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.945471966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11102024-02-03T17:23:01.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.94554749Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11112024-02-03T17:23:01.794Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.94582787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11122024-02-03T17:23:01.794Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.945876633Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11132024-02-03T17:23:01.794Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.94594187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11142024-02-03T17:23:01.795Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.945987148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11152024-02-03T17:23:01.795Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:00.946036352Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11162024-02-03T17:23:01.795Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.946096141Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11172024-02-03T17:23:01.795Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:00.946145159Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11182024-02-03T17:23:01.795Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:00.946256973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11192024-02-03T17:23:01.795Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.94630042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11202024-02-03T17:23:01.795Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.946366737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11212024-02-03T17:23:01.795Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.946429023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11222024-02-03T17:23:01.795Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.947080204Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11232024-02-03T17:23:01.795Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.947165213Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11242024-02-03T17:23:01.795Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.94734204Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11252024-02-03T17:23:01.795Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.947377785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11262024-02-03T17:23:01.795Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.948095398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11272024-02-03T17:23:01.795Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.948258833Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11282024-02-03T17:23:01.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.948873347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11292024-02-03T17:23:01.799Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.948961281Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11302024-02-03T17:23:01.799Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.948991762Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11312024-02-03T17:23:01.799Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.949041629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11322024-02-03T17:23:01.799Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.949101486Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11332024-02-03T17:23:01.799Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:00.949145264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11342024-02-03T17:23:02.791Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.94696357Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11352024-02-03T17:23:02.797Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.947169037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11362024-02-03T17:23:02.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.948321355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11372024-02-03T17:23:02.797Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.948403632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11382024-02-03T17:23:02.798Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.949107955Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11392024-02-03T17:23:02.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.949963655Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11402024-02-03T17:23:02.798Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950017437Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11412024-02-03T17:23:02.798Z{"msg":"[0]R flush_numbers[0..12]: [10, 16, 12, 1, 1, 1, 10, 10, 8, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950044365Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11422024-02-03T17:23:02.798Z{"msg":"[0]R generation[0..12]: [6, 9, 7, 1, 1, 1, 6, 6, 6, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950067364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11432024-02-03T17:23:02.798Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950110863Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11442024-02-03T17:23:02.798Z{"msg":"[1]R flush_numbers[0..12]: [10, 16, 12, 1, 1, 1, 10, 10, 8, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950136288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11452024-02-03T17:23:02.798Z{"msg":"[1]R generation[0..12]: [6, 9, 7, 1, 1, 1, 6, 6, 6, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950158937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11462024-02-03T17:23:02.798Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950182448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11472024-02-03T17:23:02.798Z{"msg":"[2]R flush_numbers[0..12]: [10, 16, 12, 1, 1, 1, 10, 10, 8, 3, 3, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950205193Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11482024-02-03T17:23:02.798Z{"msg":"[2]R generation[0..12]: [6, 9, 7, 1, 1, 1, 6, 6, 6, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950227091Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11492024-02-03T17:23:02.798Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950269657Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11502024-02-03T17:23:02.798Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950293481Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11512024-02-03T17:23:02.798ZWait for a query_work_queue command to finish before sending IO
11522024-02-03T17:23:02.798Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950338117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11532024-02-03T17:23:02.798Z{"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950361439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11542024-02-03T17:23:02.798Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950402786Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11552024-02-03T17:23:02.798Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950427114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11562024-02-03T17:23:02.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950466596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11572024-02-03T17:23:02.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.95049023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11582024-02-03T17:23:02.798Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950511966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11592024-02-03T17:23:02.798Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.95053566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11602024-02-03T17:23:02.799Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950556219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11612024-02-03T17:23:02.799Z{"msg":"ed624788-6c86-4739-b213-1f4396bdd2a4 is now active with session: a1b3ec91-f393-4cce-8c07-da724301261a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950577292Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11622024-02-03T17:23:02.799Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950597071Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169,"session_id":"a1b3ec91-f393-4cce-8c07-da724301261a"}
11632024-02-03T17:23:02.799Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:01.950617354Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169}
11642024-02-03T17:23:02.799ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11652024-02-03T17:23:02.799ZLoading write count information from file "/tmp/test_repair_verify.data"
11662024-02-03T17:23:02.799ZRun Repair workload
11672024-02-03T17:23:02.802Z01/30 Write block 55 len 8192 data: 2 2
11682024-02-03T17:23:02.803Z02/30 Write block 339 len 36864 data: 2 2 2 2 2 2 2 2 2
11692024-02-03T17:23:02.803Z03/30 Read block 108 len 12288
11702024-02-03T17:23:02.803Z04/30 Read block 20 len 12288
11712024-02-03T17:23:02.806Z05/30 Read block 348 len 40960
11722024-02-03T17:23:02.806Z06/30 Write block 574 len 24576 data: 2 2 2 3 3 4
11732024-02-03T17:23:02.809Z07/30 Write block 271 len 20480 data: 2 2 2 2 3
11742024-02-03T17:23:02.809Z08/30 Write block 74 len 20480 data: 2 2 2 2 2
11752024-02-03T17:23:02.809Z09/30 Write block 527 len 28672 data: 3 2 3 3 2 2 2
11762024-02-03T17:23:02.814Z10/30 Write block 117 len 12288 data: 2 2 2
11772024-02-03T17:23:02.814Z11/30 Write block 31 len 40960 data: 2 2 2 2 2 2 2 2 2 2
11782024-02-03T17:23:02.814Z12/30 Flush
11792024-02-03T17:23:02.820Z13/30 Write block 117 len 28672 data: 3 3 3 2 2 2 2
11802024-02-03T17:23:02.823Z14/30 Read block 444 len 16384
11812024-02-03T17:23:02.823Z15/30 Flush
11822024-02-03T17:23:02.827Z16/30 Write block 83 len 32768 data: 2 2 2 2 2 2 2 2
11832024-02-03T17:23:02.830Z17/30 Read block 478 len 28672
11842024-02-03T17:23:02.830Z18/30 Read block 3 len 24576
11852024-02-03T17:23:02.833Z19/30 Flush
11862024-02-03T17:23:02.833Z20/30 Write block 109 len 40960 data: 2 2 2 2 2 2 2 2 4 4
11872024-02-03T17:23:02.840Z21/30 Read block 378 len 24576
11882024-02-03T17:23:02.847Z22/30 Write block 374 len 32768 data: 3 2 3 3 3 2 2 2
11892024-02-03T17:23:02.847Z23/30 Write block 395 len 36864 data: 4 4 3 3 3 3 2 2 2
11902024-02-03T17:23:02.847Z24/30 Write block 184 len 24576 data: 2 2 2 2 2 2
11912024-02-03T17:23:02.847Z25/30 Write block 367 len 32768 data: 4 4 3 3 3 3 3 4
11922024-02-03T17:23:02.847Z26/30 Write block 360 len 28672 data: 2 2 3 3 3 3 3
11932024-02-03T17:23:02.851Z27/30 Write block 224 len 20480 data: 3 3 3 3 3
11942024-02-03T17:23:02.851Z28/30 Write block 136 len 24576 data: 4 4 4 4 3 3
11952024-02-03T17:23:02.851Z29/30 Read block 581 len 4096
11962024-02-03T17:23:02.851Z30/30 Read block 302 len 8192
11972024-02-03T17:23:02.851Z----------------------------------------------------------------
11982024-02-03T17:23:02.851Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:15
11992024-02-03T17:23:02.851ZStates: Active Active Active
12002024-02-03T17:23:02.851ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
12012024-02-03T17:23:02.851Z 16 Acked 1015 Write 8 Done Done Done false
12022024-02-03T17:23:02.851Z 17 Acked 1016 Read 7 Done Sent Done false
12032024-02-03T17:23:02.851Z 18 Acked 1017 Read 6 Done Sent Done false
12042024-02-03T17:23:02.851Z 19 Acked 1018 Flush 0 Done Sent Done false
12052024-02-03T17:23:02.851Z 20 Acked 1019 Write 10 Done Sent Done false
12062024-02-03T17:23:02.851Z 21 Acked 1020 Read 6 Done Sent Done false
12072024-02-03T17:23:02.851Z 22 Acked 1021 Write 8 Done Sent Done false
12082024-02-03T17:23:02.851Z 23 Acked 1022 Write 9 Done Sent Done false
12092024-02-03T17:23:02.851Z 24 Acked 1023 Write 6 Done Sent Done false
12102024-02-03T17:23:02.851Z 25 Acked 1024 Write 8 Done Sent Done false
12112024-02-03T17:23:02.851Z 26 Acked 1025 Write 7 Done Sent Done false
12122024-02-03T17:23:02.852Z 27 Acked 1026 Write 5 Done Sent Done false
12132024-02-03T17:23:02.852Z 28 Acked 1027 Write 6 Done Sent Done false
12142024-02-03T17:23:02.852Z 29 Acked 1028 Read 1 Done Sent Done false
12152024-02-03T17:23:02.852Z 30 Acked 1029 Read 2 Done Sent Sent false
12162024-02-03T17:23:02.852Z STATES DS:0 DS:1 DS:2 TOTAL
12172024-02-03T17:23:02.852Z New 0 0 0 0
12182024-02-03T17:23:02.852Z Sent 0 14 1 15
12192024-02-03T17:23:02.852Z Done 15 1 14 30
12202024-02-03T17:23:02.852Z Skipped 0 0 0 0
12212024-02-03T17:23:02.852Z Error 0 0 0 0
12222024-02-03T17:23:02.852ZLast Flush: 1018 1014 1018
12232024-02-03T17:23:02.852ZDownstairs last five completed: 1014 1013 1012 1011 1010
12242024-02-03T17:23:02.852ZUpstairs last five completed: 30 29 28 27 26
12252024-02-03T17:23:02.852Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:02.006145177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1169}
12262024-02-03T17:23:02.852ZWrote out file "/tmp/test_repair_verify.data"
12272024-02-03T17:23:02.858Z
12282024-02-03T17:23:02.865ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12292024-02-03T17:23:02.894ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12302024-02-03T17:23:02.894Z 0 000-019 6 6 6 10 10 10 F F F
12312024-02-03T17:23:02.894Z 1 020-039 10 10 10 17 17 17 F F F
12322024-02-03T17:23:02.894Z 2 040-059 10 10 10 17 17 17 F F F
12332024-02-03T17:23:02.894Z 3 060-079 10 10 10 17 17 17 F F F
12342024-02-03T17:23:02.899Z 4 080-099 10 1 10 19 1 19 F T F <---
12352024-02-03T17:23:02.899Z 5 100-119 10 10 10 18 18 18 T F T <---
12362024-02-03T17:23:02.899Z 6 120-139 10 10 10 18 18 18 T F T <---
12372024-02-03T17:23:02.899Z 7 140-159 6 6 6 10 10 10 T F T <---
12382024-02-03T17:23:02.899Z 8 160-179 6 6 6 8 8 8 F F F
12392024-02-03T17:23:02.899Z 9 180-199 4 4 4 3 3 3 T F T <---
12402024-02-03T17:23:02.899Z 10 200-219 4 4 4 3 3 3 F F F
12412024-02-03T17:23:02.899Z 11 220-239 5 5 5 7 7 7 T F T <---
12422024-02-03T17:23:02.899Z 12 240-259 1 1 1 1 1 1 F F F
12432024-02-03T17:23:02.899Z 13 260-279 10 10 10 17 17 17 F F F
12442024-02-03T17:23:02.899Z 14 280-299 6 6 6 8 8 8 F F F
12452024-02-03T17:23:02.899Z 15 300-319 1 1 1 1 1 1 F F F
12462024-02-03T17:23:02.899Z 16 320-339 10 10 10 17 17 17 F F F
12472024-02-03T17:23:02.899Z 17 340-359 10 10 10 17 17 17 F F F
12482024-02-03T17:23:02.899Z 18 360-379 9 9 9 16 16 16 T F T <---
12492024-02-03T17:23:02.899Z 19 380-399 9 9 9 16 16 16 T F T <---
12502024-02-03T17:23:02.899Z 20 400-419 8 8 8 13 13 13 T F T <---
12512024-02-03T17:23:02.899Z 21 420-439 9 9 9 16 16 16 F F F
12522024-02-03T17:23:02.899Z 22 440-459 7 7 7 12 12 12 F F F
12532024-02-03T17:23:02.899Z 23 460-479 9 9 9 16 16 16 F F F
12542024-02-03T17:23:02.900Z 24 480-499 4 4 4 5 5 5 F F F
12552024-02-03T17:23:02.900Z 25 500-519 4 4 4 6 6 6 F F F
12562024-02-03T17:23:02.900Z 26 520-539 10 10 10 17 17 17 F F F
12572024-02-03T17:23:02.900Z 27 540-559 9 9 9 16 16 16 F F F
12582024-02-03T17:23:02.900Z 28 560-579 10 10 10 17 17 17 F F F
12592024-02-03T17:23:02.900Z 29 580-599 8 8 8 14 14 14 F F F
12602024-02-03T17:23:02.900ZMax gen: 10, Max flush: 19
12612024-02-03T17:23:02.900ZError: Difference in extent metadata found!
12622024-02-03T17:23:02.900ZA Difference in extent metadata is expected here
12632024-02-03T17:23:02.900ZOn loop 5
12642024-02-03T17:23:02.900Z
12652024-02-03T17:23:02.905ZVerifying data now
12662024-02-03T17:23:07.762ZNo change after verify
12672024-02-03T17:23:07.762ZLoop: 5 Downstairs dump after verify (and repair):
12682024-02-03T17:23:07.788ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12692024-02-03T17:23:07.789Z 0 000-019 6 6 6 10 10 10 F F F
12702024-02-03T17:23:07.789Z 1 020-039 10 10 10 17 17 17 F F F
12712024-02-03T17:23:07.789Z 2 040-059 10 10 10 17 17 17 F F F
12722024-02-03T17:23:07.789Z 3 060-079 10 10 10 17 17 17 F F F
12732024-02-03T17:23:07.792Z 4 080-099 10 10 10 19 19 19 F F F
12742024-02-03T17:23:07.792Z 5 100-119 11 11 11 20 20 20 F F F
12752024-02-03T17:23:07.793Z 6 120-139 11 11 11 20 20 20 F F F
12762024-02-03T17:23:07.793Z 7 140-159 11 11 11 20 20 20 F F F
12772024-02-03T17:23:07.793Z 8 160-179 6 6 6 8 8 8 F F F
12782024-02-03T17:23:07.793Z 9 180-199 11 11 11 20 20 20 F F F
12792024-02-03T17:23:07.793Z 10 200-219 4 4 4 3 3 3 F F F
12802024-02-03T17:23:07.793Z 11 220-239 11 11 11 20 20 20 F F F
12812024-02-03T17:23:07.793Z 12 240-259 1 1 1 1 1 1 F F F
12822024-02-03T17:23:07.793Z 13 260-279 10 10 10 17 17 17 F F F
12832024-02-03T17:23:07.793Z 14 280-299 6 6 6 8 8 8 F F F
12842024-02-03T17:23:07.793Z 15 300-319 1 1 1 1 1 1 F F F
12852024-02-03T17:23:07.793Z 16 320-339 10 10 10 17 17 17 F F F
12862024-02-03T17:23:07.793Z 17 340-359 10 10 10 17 17 17 F F F
12872024-02-03T17:23:07.793Z 18 360-379 11 11 11 20 20 20 F F F
12882024-02-03T17:23:07.793Z 19 380-399 11 11 11 20 20 20 F F F
12892024-02-03T17:23:07.793Z 20 400-419 11 11 11 20 20 20 F F F
12902024-02-03T17:23:07.793Z 21 420-439 9 9 9 16 16 16 F F F
12912024-02-03T17:23:07.793Z 22 440-459 7 7 7 12 12 12 F F F
12922024-02-03T17:23:07.793Z 23 460-479 9 9 9 16 16 16 F F F
12932024-02-03T17:23:07.793Z 24 480-499 4 4 4 5 5 5 F F F
12942024-02-03T17:23:07.793Z 25 500-519 4 4 4 6 6 6 F F F
12952024-02-03T17:23:07.793Z 26 520-539 10 10 10 17 17 17 F F F
12962024-02-03T17:23:07.793Z 27 540-559 9 9 9 16 16 16 F F F
12972024-02-03T17:23:07.793Z 28 560-579 10 10 10 17 17 17 F F F
12982024-02-03T17:23:07.793Z 29 580-599 8 8 8 14 14 14 F F F
12992024-02-03T17:23:07.793ZMax gen: 11, Max flush: 20
13002024-02-03T17:23:07.794Z
13012024-02-03T17:23:07.794ZBegin loop 6
13022024-02-03T17:23:07.794ZDownstairs to restart: 2
13032024-02-03T17:23:07.827Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.982394204Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177}
13042024-02-03T17:23:07.827Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.982896646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177}
13052024-02-03T17:23:07.827Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.982943128Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177}
13062024-02-03T17:23:07.827Z{"msg":"Crucible 407eb8a4-a379-4864-8bc0-483f0b1bd011 has session id: ef7fbe61-ed8c-4ea5-af44-e9aa08861c42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.982980909Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13072024-02-03T17:23:07.832Z{"msg":"Upstairs opts: Upstairs UUID: 407eb8a4-a379-4864-8bc0-483f0b1bd011, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983035265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13082024-02-03T17:23:07.833Z{"msg":"Crucible stats registered with UUID: 407eb8a4-a379-4864-8bc0-483f0b1bd011","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983078186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13092024-02-03T17:23:07.833ZCrucible runtime is spawned
13102024-02-03T17:23:07.833Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983358005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177}
13112024-02-03T17:23:07.833Z{"msg":"407eb8a4-a379-4864-8bc0-483f0b1bd011 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983618207Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13122024-02-03T17:23:07.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983670306Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13132024-02-03T17:23:07.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983709416Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13142024-02-03T17:23:07.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983753799Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13152024-02-03T17:23:07.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983862241Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13162024-02-03T17:23:07.833Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.983936844Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13172024-02-03T17:23:07.833Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.984008418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13182024-02-03T17:23:07.833Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.984053647Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13192024-02-03T17:23:07.833Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.984125804Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13202024-02-03T17:23:07.833Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.984153933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13212024-02-03T17:23:07.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.984230946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13222024-02-03T17:23:07.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.984589227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13232024-02-03T17:23:07.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.984669075Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13242024-02-03T17:23:07.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.985216773Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13252024-02-03T17:23:07.833Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.985277683Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13262024-02-03T17:23:07.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.985303936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13272024-02-03T17:23:07.833Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.985387541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13282024-02-03T17:23:07.833Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.986170829Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13292024-02-03T17:23:07.833Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.986377921Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13302024-02-03T17:23:07.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.98655621Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13312024-02-03T17:23:07.834Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.986604487Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13322024-02-03T17:23:07.834Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.987119881Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13332024-02-03T17:23:07.834Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.987279798Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13342024-02-03T17:23:07.834Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.987397228Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13352024-02-03T17:23:07.834Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.987421534Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13362024-02-03T17:23:07.834Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.987451966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13372024-02-03T17:23:07.842Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.987595618Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13382024-02-03T17:23:07.842Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.987673435Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13392024-02-03T17:23:07.842Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988355361Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13402024-02-03T17:23:07.842Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988419622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13412024-02-03T17:23:07.842Z{"msg":"[0]R flush_numbers[0..12]: [10, 17, 17, 17, 19, 20, 20, 20, 8, 20, 3, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988447097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13422024-02-03T17:23:07.842Z{"msg":"[0]R generation[0..12]: [6, 10, 10, 10, 10, 11, 11, 11, 6, 11, 4, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988490345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13432024-02-03T17:23:07.842Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988516739Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13442024-02-03T17:23:07.842Z{"msg":"[1]R flush_numbers[0..12]: [10, 17, 17, 17, 19, 20, 20, 20, 8, 20, 3, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988559525Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13452024-02-03T17:23:07.842Z{"msg":"[1]R generation[0..12]: [6, 10, 10, 10, 10, 11, 11, 11, 6, 11, 4, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988585194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13462024-02-03T17:23:07.843Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
13472024-02-03T17:23:07.843Ztime":"2024-02-03T17:23:06.988610366Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13482024-02-03T17:23:07.843Z{"msg":"[2]R flush_numbers[0..12]: [10, 17, 17, 17, 19, 20, 20, 20, 8, 20, 3, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988748943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13492024-02-03T17:23:07.843Z{"msg":"[2]R generation[0..12]: [6, 10, 10, 10, 10, 11, 11, 11, 6, 11, 4, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.98879918Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13502024-02-03T17:23:07.843Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988826187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13512024-02-03T17:23:07.843Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.98886721Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13522024-02-03T17:23:07.843Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988891072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13532024-02-03T17:23:07.843Z{"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988912686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13542024-02-03T17:23:07.843Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988952822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13552024-02-03T17:23:07.843Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988976901Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13562024-02-03T17:23:07.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.988997009Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13572024-02-03T17:23:07.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.98906064Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13582024-02-03T17:23:07.843Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.989124271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13592024-02-03T17:23:07.844Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.989149111Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13602024-02-03T17:23:07.844Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.989175999Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13612024-02-03T17:23:07.844Z{"msg":"407eb8a4-a379-4864-8bc0-483f0b1bd011 is now active with session: ef7fbe61-ed8c-4ea5-af44-e9aa08861c42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.989242766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13622024-02-03T17:23:07.844Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.989268143Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177,"session_id":"ef7fbe61-ed8c-4ea5-af44-e9aa08861c42"}
13632024-02-03T17:23:07.844Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:06.989331076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177}
13642024-02-03T17:23:07.844ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13652024-02-03T17:23:07.844ZLoading write count information from file "/tmp/test_repair_verify.data"
13662024-02-03T17:23:07.844ZRun Repair workload
13672024-02-03T17:23:07.844Z01/30 Write block 56 len 16384 data: 3 2 2 2
13682024-02-03T17:23:07.844Z02/30 Write block 31 len 8192 data: 3 3
13692024-02-03T17:23:07.844Z03/30 Read block 25 len 32768
13702024-02-03T17:23:07.848Z04/30 Read block 25 len 40960
13712024-02-03T17:23:07.848Z05/30 Read block 278 len 40960
13722024-02-03T17:23:07.848Z06/30 Write block 474 len 28672 data: 2 2 2 3 3 3 3
13732024-02-03T17:23:07.848Z07/30 Read block 143 len 28672
13742024-02-03T17:23:07.848Z08/30 Read block 91 len 36864
13752024-02-03T17:23:07.849Z09/30 Read block 232 len 28672
13762024-02-03T17:23:07.852Z10/30 Read block 380 len 40960
13772024-02-03T17:23:07.852Z11/30 Read block 135 len 36864
13782024-02-03T17:23:07.855Z12/30 Write block 205 len 4096 data: 2
13792024-02-03T17:23:07.856Z13/30 Flush
13802024-02-03T17:23:07.859Z14/30 Write block 398 len 8192 data: 4 4
13812024-02-03T17:23:07.859Z15/30 Read block 185 len 12288
13822024-02-03T17:23:07.862Z16/30 Write block 495 len 36864 data: 2 2 2 2 2 2 2 2 2
13832024-02-03T17:23:07.862Z17/30 Read block 280 len 40960
13842024-02-03T17:23:07.866Z18/30 Read block 318 len 12288
13852024-02-03T17:23:07.866Z19/30 Read block 58 len 20480
13862024-02-03T17:23:07.866Z20/30 Read block 400 len 32768
13872024-02-03T17:23:07.869Z21/30 Read block 464 len 28672
13882024-02-03T17:23:07.869Z22/30 Read block 557 len 32768
13892024-02-03T17:23:07.872Z23/30 Write block 270 len 36864 data: 2 3 3 3 3 4 3 3 2
13902024-02-03T17:23:07.873Z24/30 Write block 210 len 24576 data: 2 2 2 2 2 2
13912024-02-03T17:23:07.876Z25/30 Write block 176 len 36864 data: 2 2 2 2 2 2 2 2 3
13922024-02-03T17:23:07.876Z26/30 Read block 111 len 28672
13932024-02-03T17:23:07.879Z27/30 Write block 440 len 16384 data: 3 3 3 3
13942024-02-03T17:23:07.879Z28/30 Write block 506 len 16384 data: 3 3 3 3
13952024-02-03T17:23:07.879Z29/30 Read block 14 len 32768
13962024-02-03T17:23:07.883Z30/30 Write block 449 len 8192 data: 5 5
13972024-02-03T17:23:07.884Z----------------------------------------------------------------
13982024-02-03T17:23:07.884Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
13992024-02-03T17:23:07.884ZStates: Active Active Active
14002024-02-03T17:23:07.884ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14012024-02-03T17:23:07.884Z 1 Acked 1000 Write 4 Done Done Done false
14022024-02-03T17:23:07.884Z 2 Acked 1001 Write 2 Done Done Sent false
14032024-02-03T17:23:07.884Z 3 Acked 1002 Read 8 Done Done Sent false
14042024-02-03T17:23:07.884Z 4 Acked 1003 Read 10 Done Done Sent false
14052024-02-03T17:23:07.884Z 5 Acked 1004 Read 10 Done Done Sent false
14062024-02-03T17:23:07.884Z 6 Acked 1005 Write 7 Done Done Sent false
14072024-02-03T17:23:07.884Z 7 Acked 1006 Read 7 Done Done Sent false
14082024-02-03T17:23:07.884Z 8 Acked 1007 Read 9 Done Done Sent false
14092024-02-03T17:23:07.885Z 9 Acked 1008 Read 7 Done Done Sent false
14102024-02-03T17:23:07.885Z 10 Acked 1009 Read 10 Done Done Sent false
14112024-02-03T17:23:07.885Z 11 Acked 1010 Read 9 Done Done Sent false
14122024-02-03T17:23:07.885Z 12 Acked 1011 Write 1 Done Done Sent false
14132024-02-03T17:23:07.885Z 13 Acked 1012 Flush 0 Done Done Sent false
14142024-02-03T17:23:07.885Z 14 Acked 1013 Write 2 Done Done Sent false
14152024-02-03T17:23:07.885Z 15 Acked 1014 Read 3 Done Done Sent false
14162024-02-03T17:23:07.885Z 16 Acked 1015 Write 9 Done Done Sent false
14172024-02-03T17:23:07.885Z 17 Acked 1016 Read 10 Done Done Sent false
14182024-02-03T17:23:07.885Z 18 Acked 1017 Read 3 Done Done Sent false
14192024-02-03T17:23:07.885Z 19 Acked 1018 Read 5 Done Done Sent false
14202024-02-03T17:23:07.885Z 20 Acked 1019 Read 8 Done Done Sent false
14212024-02-03T17:23:07.885Z 21 Acked 1020 Read 7 Done Done Sent false
14222024-02-03T17:23:07.886Z 22 Acked 1021 Read 8 Done Done Sent false
14232024-02-03T17:23:07.886Z 23 Acked 1022 Write 9 Done Done Sent false
14242024-02-03T17:23:07.886Z 24 Acked 1023 Write 6 Done Done Sent false
14252024-02-03T17:23:07.886Z 25 Acked 1024 Write 9 Done Done Sent false
14262024-02-03T17:23:07.886Z 26 Acked 1025 Read 7 Done Done Sent false
14272024-02-03T17:23:07.886Z 27 Acked 1026 Write 4 Done Done Sent false
14282024-02-03T17:23:07.886Z 28 Acked 1027 Write 4 Done Done Sent false
14292024-02-03T17:23:07.886Z 29 Acked 1028 Read 8 Done Done Sent false
14302024-02-03T17:23:07.886Z 30 Acked 1029 Write 2 Sent Sent Sent false
14312024-02-03T17:23:07.886Z STATES DS:0 DS:1 DS:2 TOTAL
14322024-02-03T17:23:07.886Z New 0 0 0 0
14332024-02-03T17:23:07.886Z Sent 1 1 29 31
14342024-02-03T17:23:07.886Z Done 29 29 1 59
14352024-02-03T17:23:07.887Z Skipped 0 0 0 0
14362024-02-03T17:23:07.887Z Error 0 0 0 0
14372024-02-03T17:23:07.887ZLast Flush: 1012 1012 0
14382024-02-03T17:23:07.887ZDownstairs last five completed:
14392024-02-03T17:23:07.887ZUpstairs last five completed: 30 29 28 27 26
14402024-02-03T17:23:07.887Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:07.038532922Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1177}
14412024-02-03T17:23:07.887ZWrote out file "/tmp/test_repair_verify.data"
14422024-02-03T17:23:07.891Z
14432024-02-03T17:23:07.900ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14442024-02-03T17:23:07.927ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14452024-02-03T17:23:07.928Z 0 000-019 6 6 6 10 10 10 F F F
14462024-02-03T17:23:07.931Z 1 020-039 12 12 10 21 21 17 F F F <---
14472024-02-03T17:23:07.931Z 2 040-059 12 12 10 21 21 17 F F T <---
14482024-02-03T17:23:07.931Z 3 060-079 10 10 10 17 17 17 F F F
14492024-02-03T17:23:07.931Z 4 080-099 10 10 10 19 19 19 F F F
14502024-02-03T17:23:07.931Z 5 100-119 11 11 11 20 20 20 F F F
14512024-02-03T17:23:07.931Z 6 120-139 11 11 11 20 20 20 F F F
14522024-02-03T17:23:07.931Z 7 140-159 11 11 11 20 20 20 F F F
14532024-02-03T17:23:07.932Z 8 160-179 6 6 6 8 8 8 T T F <---
14542024-02-03T17:23:07.932Z 9 180-199 11 11 11 20 20 20 T T F <---
14552024-02-03T17:23:07.932Z 10 200-219 12 12 4 21 21 3 T T F <---
14562024-02-03T17:23:07.932Z 11 220-239 11 11 11 20 20 20 F F F
14572024-02-03T17:23:07.932Z 12 240-259 1 1 1 1 1 1 F F F
14582024-02-03T17:23:07.932Z 13 260-279 10 10 10 17 17 17 T T F <---
14592024-02-03T17:23:07.932Z 14 280-299 6 6 6 8 8 8 F F F
14602024-02-03T17:23:07.932Z 15 300-319 1 1 1 1 1 1 F F F
14612024-02-03T17:23:07.932Z 16 320-339 10 10 10 17 17 17 F F F
14622024-02-03T17:23:07.932Z 17 340-359 10 10 10 17 17 17 F F F
14632024-02-03T17:23:07.932Z 18 360-379 11 11 11 20 20 20 F F F
14642024-02-03T17:23:07.932Z 19 380-399 11 11 11 20 20 20 T T F <---
14652024-02-03T17:23:07.932Z 20 400-419 11 11 11 20 20 20 F F F
14662024-02-03T17:23:07.932Z 21 420-439 9 9 9 16 16 16 F F F
14672024-02-03T17:23:07.932Z 22 440-459 7 7 7 12 12 12 T T F <---
14682024-02-03T17:23:07.932Z 23 460-479 12 12 9 21 21 16 F F F <---
14692024-02-03T17:23:07.932Z 24 480-499 12 12 4 21 21 5 T T F <---
14702024-02-03T17:23:07.932Z 25 500-519 4 4 4 6 6 6 T T F <---
14712024-02-03T17:23:07.932Z 26 520-539 10 10 10 17 17 17 F F F
14722024-02-03T17:23:07.932Z 27 540-559 9 9 9 16 16 16 F F F
14732024-02-03T17:23:07.933Z 28 560-579 10 10 10 17 17 17 F F F
14742024-02-03T17:23:07.933Z 29 580-599 8 8 8 14 14 14 F F F
14752024-02-03T17:23:07.933ZMax gen: 12, Max flush: 21
14762024-02-03T17:23:07.933ZError: Difference in extent metadata found!
14772024-02-03T17:23:07.936ZA Difference in extent metadata is expected here
14782024-02-03T17:23:07.936ZOn loop 6
14792024-02-03T17:23:07.936Z
14802024-02-03T17:23:07.936ZVerifying data now
14812024-02-03T17:23:12.958ZNo change after verify
14822024-02-03T17:23:12.958ZLoop: 6 Downstairs dump after verify (and repair):
14832024-02-03T17:23:12.983ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14842024-02-03T17:23:12.983Z 0 000-019 6 6 6 10 10 10 F F F
14852024-02-03T17:23:12.987Z 1 020-039 12 12 12 21 21 21 F F F
14862024-02-03T17:23:12.987Z 2 040-059 12 12 12 21 21 21 F F F
14872024-02-03T17:23:12.987Z 3 060-079 10 10 10 17 17 17 F F F
14882024-02-03T17:23:12.987Z 4 080-099 10 10 10 19 19 19 F F F
14892024-02-03T17:23:12.987Z 5 100-119 11 11 11 20 20 20 F F F
14902024-02-03T17:23:12.987Z 6 120-139 11 11 11 20 20 20 F F F
14912024-02-03T17:23:12.988Z 7 140-159 11 11 11 20 20 20 F F F
14922024-02-03T17:23:12.988Z 8 160-179 13 13 13 22 22 22 F F F
14932024-02-03T17:23:12.988Z 9 180-199 13 13 13 22 22 22 F F F
14942024-02-03T17:23:12.988Z 10 200-219 13 13 13 22 22 22 F F F
14952024-02-03T17:23:12.988Z 11 220-239 11 11 11 20 20 20 F F F
14962024-02-03T17:23:12.988Z 12 240-259 1 1 1 1 1 1 F F F
14972024-02-03T17:23:12.988Z 13 260-279 13 13 13 22 22 22 F F F
14982024-02-03T17:23:12.988Z 14 280-299 6 6 6 8 8 8 F F F
14992024-02-03T17:23:12.988Z 15 300-319 1 1 1 1 1 1 F F F
15002024-02-03T17:23:12.988Z 16 320-339 10 10 10 17 17 17 F F F
15012024-02-03T17:23:12.988Z 17 340-359 10 10 10 17 17 17 F F F
15022024-02-03T17:23:12.988Z 18 360-379 11 11 11 20 20 20 F F F
15032024-02-03T17:23:12.988Z 19 380-399 13 13 13 22 22 22 F F F
15042024-02-03T17:23:12.989Z 20 400-419 11 11 11 20 20 20 F F F
15052024-02-03T17:23:12.989Z 21 420-439 9 9 9 16 16 16 F F F
15062024-02-03T17:23:12.989Z 22 440-459 13 13 13 22 22 22 F F F
15072024-02-03T17:23:12.989Z 23 460-479 12 12 12 21 21 21 F F F
15082024-02-03T17:23:12.989Z 24 480-499 13 13 13 22 22 22 F F F
15092024-02-03T17:23:12.989Z 25 500-519 13 13 13 22 22 22 F F F
15102024-02-03T17:23:12.989Z 26 520-539 10 10 10 17 17 17 F F F
15112024-02-03T17:23:12.989Z 27 540-559 9 9 9 16 16 16 F F F
15122024-02-03T17:23:12.989Z 28 560-579 10 10 10 17 17 17 F F F
15132024-02-03T17:23:12.989Z 29 580-599 8 8 8 14 14 14 F F F
15142024-02-03T17:23:12.989ZMax gen: 13, Max flush: 22
15152024-02-03T17:23:12.993Z
15162024-02-03T17:23:12.994ZBegin loop 7
15172024-02-03T17:23:12.994ZDownstairs to restart: 0
15182024-02-03T17:23:13.028Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.177967611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185}
15192024-02-03T17:23:13.029Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.178637496Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185}
15202024-02-03T17:23:13.061Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.178764076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185}
15212024-02-03T17:23:13.061Z{"msg":"Crucible ccba58be-17e0-41ea-b60d-c69a13127b32 has session id: 0722a232-55a7-4e09-809a-fa8648dc4f5c","v":0,Crucible runtime is spawned
15222024-02-03T17:23:13.061Z"name":"crucible","level":30,"time":"2024-02-03T17:23:12.17894545Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15232024-02-03T17:23:13.061Z{"msg":"Upstairs opts: Upstairs UUID: ccba58be-17e0-41ea-b60d-c69a13127b32, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179041731Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15242024-02-03T17:23:13.062Z{"msg":"Crucible stats registered with UUID: ccba58be-17e0-41ea-b60d-c69a13127b32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179120654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15252024-02-03T17:23:13.062Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179145364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185}
15262024-02-03T17:23:13.062Z{"msg":"ccba58be-17e0-41ea-b60d-c69a13127b32 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179185232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15272024-02-03T17:23:13.062Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179208089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15282024-02-03T17:23:13.062Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179251139Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15292024-02-03T17:23:13.062Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179275486Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15302024-02-03T17:23:13.062Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179317885Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15312024-02-03T17:23:13.062Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179342104Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15322024-02-03T17:23:13.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179381526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15332024-02-03T17:23:13.063Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179446004Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15342024-02-03T17:23:13.063Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179479486Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15352024-02-03T17:23:13.063Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179519636Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15362024-02-03T17:23:13.063Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.17969028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15372024-02-03T17:23:13.063Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.179939062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15382024-02-03T17:23:13.063Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.180080031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15392024-02-03T17:23:13.063Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.181233906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15402024-02-03T17:23:13.063Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.181360167Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15412024-02-03T17:23:13.063Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.181445293Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15422024-02-03T17:23:13.063Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.181492125Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15432024-02-03T17:23:13.064Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.182453802Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15442024-02-03T17:23:13.064Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.182669249Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15452024-02-03T17:23:13.064Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.182843015Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15462024-02-03T17:23:13.064Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.18288491Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15472024-02-03T17:23:13.064Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.183660657Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15482024-02-03T17:23:13.064Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.183734836Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15492024-02-03T17:23:13.064Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.183780802Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15502024-02-03T17:23:13.064Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.18384868Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15512024-02-03T17:23:13.065Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.1838943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15522024-02-03T17:23:13.065Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.183934373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15532024-02-03T17:23:13.065Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.183992786Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15542024-02-03T17:23:13.065Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.184708985Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15552024-02-03T17:23:13.065Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.184786066Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15562024-02-03T17:23:13.065Z{"msg":"[0]R flush_numbers[0..12]: [10, 21, 21, 17, 19, 20, 20, 20, 22, 22, 22, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.184813998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15572024-02-03T17:23:13.065Z{"msg":"[0]R generation[0..12]: [6, 12, 12, 10, 10, 11, 11, 11, 13, 13, 13, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.184837874Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15582024-02-03T17:23:13.065Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.184958526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15592024-02-03T17:23:13.065Z{"msg":"[1]R flush_numbers[0..12]: [10, 21, 21, 17, 19, 20, 20, 20, 22, 22, 22, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185011146Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15602024-02-03T17:23:13.065Z{"msg":"[1]R generation[0..12]: [6, 12, 12, 10, 10, 11, 11, 11, 13, 13, 13, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185037587Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15612024-02-03T17:23:13.066Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.18506149Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15622024-02-03T17:23:13.066Z{"msg":"[2]R flush_numbers[0..12]: [10, 21, 21, 17, 19, 20, 20, 20, 22, 22, 22, 20]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185104312Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15632024-02-03T17:23:13.066Z{"msg":"[2]R generation[0..12]: [6, 12, 12, 10, 10, 11, 11, 11, 13, 13, 13, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.18512883Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15642024-02-03T17:23:13.066Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185152788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15652024-02-03T17:23:13.066Z{"msg":"Max found gen is 14","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
15662024-02-03T17:23:13.066Zlevel":30,"time":"2024-02-03T17:23:12.185178193Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15672024-02-03T17:23:13.066Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185200613Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15682024-02-03T17:23:13.066Z{"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185221851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15692024-02-03T17:23:13.067Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185251266Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15702024-02-03T17:23:13.067Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185274001Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15712024-02-03T17:23:13.067Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185314374Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15722024-02-03T17:23:13.067Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185338215Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15732024-02-03T17:23:13.067Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185388767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15742024-02-03T17:23:13.067Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.18544068Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15752024-02-03T17:23:13.067Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185463631Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15762024-02-03T17:23:13.067Z{"msg":"ccba58be-17e0-41ea-b60d-c69a13127b32 is now active with session: 0722a232-55a7-4e09-809a-fa8648dc4f5c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185493013Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15772024-02-03T17:23:13.067Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.185551718Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185,"session_id":"0722a232-55a7-4e09-809a-fa8648dc4f5c"}
15782024-02-03T17:23:13.067Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:12.18562028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185}
15792024-02-03T17:23:13.067ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15802024-02-03T17:23:13.068ZLoading write count information from file "/tmp/test_repair_verify.data"
15812024-02-03T17:23:13.068ZRun Repair workload
15822024-02-03T17:23:13.068Z01/30 Write block 118 len 16384 data: 5 4 3 3
15832024-02-03T17:23:13.068Z02/30 Read block 498 len 24576
15842024-02-03T17:23:13.068Z03/30 Write block 300 len 28672 data: 2 2 2 2 2 2 2
15852024-02-03T17:23:13.068Z04/30 Read block 392 len 16384
15862024-02-03T17:23:13.068Z05/30 Read block 462 len 40960
15872024-02-03T17:23:13.069Z06/30 Write block 221 len 20480 data: 4 4 3 4 4
15882024-02-03T17:23:13.073Z07/30 Flush
15892024-02-03T17:23:13.073Z08/30 Write block 589 len 16384 data: 2 2 3 3
15902024-02-03T17:23:13.073Z09/30 Read block 148 len 20480
15912024-02-03T17:23:13.073Z10/30 Write block 519 len 4096 data: 2
15922024-02-03T17:23:13.073Z11/30 Write block 105 len 40960 data: 2 2 2 2 3 3 3 3 3 3
15932024-02-03T17:23:13.073Z12/30 Read block 507 len 20480
15942024-02-03T17:23:13.073Z13/30 Write block 551 len 32768 data: 3 4 4 3 3 4 3 3
15952024-02-03T17:23:13.073Z14/30 Read block 21 len 24576
15962024-02-03T17:23:13.073Z15/30 Read block 513 len 4096
15972024-02-03T17:23:13.073Z16/30 Write block 234 len 32768 data: 2 2 2 2 2 2 2 2
15982024-02-03T17:23:13.073Z17/30 Write block 345 len 32768 data: 3 3 3 2 2 2 2 2
15992024-02-03T17:23:13.073Z18/30 Read block 314 len 32768
16002024-02-03T17:23:13.073Z19/30 Write block 254 len 12288 data: 2 2 2
16012024-02-03T17:23:13.073Z20/30 Write block 268 len 36864 data: 3 2 3 4 4 4 4 5 4
16022024-02-03T17:23:13.074Z21/30 Read block 84 len 20480
16032024-02-03T17:23:13.074Z22/30 Read block 295 len 24576
16042024-02-03T17:23:13.074Z23/30 Read block 293 len 20480
16052024-02-03T17:23:13.074Z24/30 Read block 279 len 20480
16062024-02-03T17:23:13.074Z25/30 Write block 512 len 4096 data: 2
16072024-02-03T17:23:13.074Z26/30 Write block 508 len 16384 data: 4 4 2 2
16082024-02-03T17:23:13.074Z27/30 Write block 365 len 20480 data: 4 4 5 5 4
16092024-02-03T17:23:13.074Z28/30 Read block 286 len 28672
16102024-02-03T17:23:13.093Z29/30 Write block 269 len 40960 data: 3 4 5 5 5 5 6 5 4 3
16112024-02-03T17:23:13.093Z30/30 Read block 510 len 24576
16122024-02-03T17:23:13.093Z----------------------------------------------------------------
16132024-02-03T17:23:13.093Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16142024-02-03T17:23:13.093ZStates: Active Active Active
16152024-02-03T17:23:13.093ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16162024-02-03T17:23:13.093Z 1 Acked 1000 Write 4 Sent Done Done false
16172024-02-03T17:23:13.093Z 2 Acked 1001 Read 6 Sent Done Done false
16182024-02-03T17:23:13.093Z 3 Acked 1002 Write 7 Sent Done Done false
16192024-02-03T17:23:13.093Z 4 Acked 1003 Read 4 Sent Done Done false
16202024-02-03T17:23:13.093Z 5 Acked 1004 Read 10 Sent Done Done false
16212024-02-03T17:23:13.093Z 6 Acked 1005 Write 5 Sent Done Done false
16222024-02-03T17:23:13.093Z 7 Acked 1006 Flush 0 Sent Done Done false
16232024-02-03T17:23:13.093Z 8 Acked 1007 Write 4 Sent Done Done false
16242024-02-03T17:23:13.093Z 9 Acked 1008 Read 5 Sent Done Done false
16252024-02-03T17:23:13.094Z 10 Acked 1009 Write 1 Sent Done Done false
16262024-02-03T17:23:13.094Z 11 Acked 1010 Write 10 Sent Done Done false
16272024-02-03T17:23:13.094Z 12 Acked 1011 Read 5 Sent Done Done false
16282024-02-03T17:23:13.094Z 13 Acked 1012 Write 8 Sent Done Done false
16292024-02-03T17:23:13.094Z 14 Acked 1013 Read 6 Sent Done Done false
16302024-02-03T17:23:13.094Z 15 Acked 1014 Read 1 Sent Done Done false
16312024-02-03T17:23:13.094Z 16 Acked 1015 Write 8 Sent Done Done false
16322024-02-03T17:23:13.094Z 17 Acked 1016 Write 8 Sent Done Done false
16332024-02-03T17:23:13.094Z 18 Acked 1017 Read 8 Sent Done Done false
16342024-02-03T17:23:13.094Z 19 Acked 1018 Write 3 Sent Done Done false
16352024-02-03T17:23:13.094Z 20 Acked 1019 Write 9 Sent Done Done false
16362024-02-03T17:23:13.094Z 21 Acked 1020 Read 5 Sent Done Done false
16372024-02-03T17:23:13.094Z 22 Acked 1021 Read 6 Sent Done Done false
16382024-02-03T17:23:13.094Z 23 Acked 1022 Read 5 Sent Done Done false
16392024-02-03T17:23:13.094Z 24 Acked 1023 Read 5 Sent Done Done false
16402024-02-03T17:23:13.094Z 25 Acked 1024 Write 1 Sent Done Done false
16412024-02-03T17:23:13.094Z 26 Acked 1025 Write 4 Sent Done Done false
16422024-02-03T17:23:13.095Z 27 Acked 1026 Write 5 Sent Done Done false
16432024-02-03T17:23:13.095Z 28 Acked 1027 Read 7 Sent Done Done false
16442024-02-03T17:23:13.095Z 29 Acked 1028 Write 10 Sent Done Done false
16452024-02-03T17:23:13.095Z 30 Acked 1029 Read 6 Sent Done Sent false
16462024-02-03T17:23:13.095Z STATES DS:0 DS:1 DS:2 TOTAL
16472024-02-03T17:23:13.095Z New 0 0 0 0
16482024-02-03T17:23:13.095Z Sent 30 0 1 31
16492024-02-03T17:23:13.095Z Done 0 30 29 59
16502024-02-03T17:23:13.095Z Skipped 0 0 0 0
16512024-02-03T17:23:13.095Z Error 0 0 0 0
16522024-02-03T17:23:13.095ZLast Flush: 0 1006 1006
16532024-02-03T17:23:13.095ZDownstairs last five completed:
16542024-02-03T17:23:13.095ZUpstairs last five completed: 30 29 28 27 26
16552024-02-03T17:23:13.095Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:12.234169044Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1185}
16562024-02-03T17:23:13.095ZWrote out file "/tmp/test_repair_verify.data"
16572024-02-03T17:23:13.095Z
16582024-02-03T17:23:13.100ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16592024-02-03T17:23:13.122ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16602024-02-03T17:23:13.122Z 0 000-019 6 6 6 10 10 10 F F F
16612024-02-03T17:23:13.122Z 1 020-039 12 12 12 21 21 21 F F F
16622024-02-03T17:23:13.122Z 2 040-059 12 12 12 21 21 21 F F F
16632024-02-03T17:23:13.122Z 3 060-079 10 10 10 17 17 17 F F F
16642024-02-03T17:23:13.122Z 4 080-099 10 10 10 19 19 19 F F F
16652024-02-03T17:23:13.122Z 5 100-119 11 14 14 20 23 23 F T T <---
16662024-02-03T17:23:13.122Z 6 120-139 11 14 14 20 23 23 F F F <---
16672024-02-03T17:23:13.126Z 7 140-159 11 11 11 20 20 20 F F F
16682024-02-03T17:23:13.126Z 8 160-179 13 13 13 22 22 22 F F F
16692024-02-03T17:23:13.126Z 9 180-199 13 13 13 22 22 22 F F F
16702024-02-03T17:23:13.126Z 10 200-219 13 13 13 22 22 22 F F F
16712024-02-03T17:23:13.126Z 11 220-239 11 14 14 20 23 23 F T T <---
16722024-02-03T17:23:13.126Z 12 240-259 1 1 1 1 1 1 F T T <---
16732024-02-03T17:23:13.126Z 13 260-279 13 13 13 22 22 22 F T T <---
16742024-02-03T17:23:13.126Z 14 280-299 6 6 6 8 8 8 F F F
16752024-02-03T17:23:13.126Z 15 300-319 1 14 14 1 23 23 F F F <---
16762024-02-03T17:23:13.126Z 16 320-339 10 10 10 17 17 17 F F F
16772024-02-03T17:23:13.126Z 17 340-359 10 10 10 17 17 17 F T T <---
16782024-02-03T17:23:13.126Z 18 360-379 11 11 11 20 20 20 F T T <---
16792024-02-03T17:23:13.126Z 19 380-399 13 13 13 22 22 22 F F F
16802024-02-03T17:23:13.126Z 20 400-419 11 11 11 20 20 20 F F F
16812024-02-03T17:23:13.126Z 21 420-439 9 9 9 16 16 16 F F F
16822024-02-03T17:23:13.126Z 22 440-459 13 13 13 22 22 22 F F F
16832024-02-03T17:23:13.126Z 23 460-479 12 12 12 21 21 21 F F F
16842024-02-03T17:23:13.126Z 24 480-499 13 13 13 22 22 22 F F F
16852024-02-03T17:23:13.127Z 25 500-519 13 13 13 22 22 22 F T T <---
16862024-02-03T17:23:13.127Z 26 520-539 10 10 10 17 17 17 F F F
16872024-02-03T17:23:13.127Z 27 540-559 9 9 9 16 16 16 F T T <---
16882024-02-03T17:23:13.127Z 28 560-579 10 10 10 17 17 17 F F F
16892024-02-03T17:23:13.127Z 29 580-599 8 8 8 14 14 14 F T T <---
16902024-02-03T17:23:13.127ZMax gen: 14, Max flush: 23
16912024-02-03T17:23:13.127ZError: Difference in extent metadata found!
16922024-02-03T17:23:13.130ZA Difference in extent metadata is expected here
16932024-02-03T17:23:13.130ZOn loop 7
16942024-02-03T17:23:13.130Z
16952024-02-03T17:23:13.143ZVerifying data now
16962024-02-03T17:23:18.141ZNo change after verify
16972024-02-03T17:23:18.142ZLoop: 7 Downstairs dump after verify (and repair):
16982024-02-03T17:23:18.164ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16992024-02-03T17:23:18.164Z 0 000-019 6 6 6 10 10 10 F F F
17002024-02-03T17:23:18.168Z 1 020-039 12 12 12 21 21 21 F F F
17012024-02-03T17:23:18.169Z 2 040-059 12 12 12 21 21 21 F F F
17022024-02-03T17:23:18.169Z 3 060-079 10 10 10 17 17 17 F F F
17032024-02-03T17:23:18.169Z 4 080-099 10 10 10 19 19 19 F F F
17042024-02-03T17:23:18.169Z 5 100-119 15 15 15 24 24 24 F F F
17052024-02-03T17:23:18.169Z 6 120-139 14 14 14 23 23 23 F F F
17062024-02-03T17:23:18.169Z 7 140-159 11 11 11 20 20 20 F F F
17072024-02-03T17:23:18.170Z 8 160-179 13 13 13 22 22 22 F F F
17082024-02-03T17:23:18.170Z 9 180-199 13 13 13 22 22 22 F F F
17092024-02-03T17:23:18.170Z 10 200-219 13 13 13 22 22 22 F F F
17102024-02-03T17:23:18.170Z 11 220-239 15 15 15 24 24 24 F F F
17112024-02-03T17:23:18.170Z 12 240-259 15 15 15 24 24 24 F F F
17122024-02-03T17:23:18.170Z 13 260-279 15 15 15 24 24 24 F F F
17132024-02-03T17:23:18.170Z 14 280-299 6 6 6 8 8 8 F F F
17142024-02-03T17:23:18.170Z 15 300-319 14 14 14 23 23 23 F F F
17152024-02-03T17:23:18.170Z 16 320-339 10 10 10 17 17 17 F F F
17162024-02-03T17:23:18.171Z 17 340-359 15 15 15 24 24 24 F F F
17172024-02-03T17:23:18.171Z 18 360-379 15 15 15 24 24 24 F F F
17182024-02-03T17:23:18.171Z 19 380-399 13 13 13 22 22 22 F F F
17192024-02-03T17:23:18.171Z 20 400-419 11 11 11 20 20 20 F F F
17202024-02-03T17:23:18.171Z 21 420-439 9 9 9 16 16 16 F F F
17212024-02-03T17:23:18.171Z 22 440-459 13 13 13 22 22 22 F F F
17222024-02-03T17:23:18.171Z 23 460-479 12 12 12 21 21 21 F F F
17232024-02-03T17:23:18.171Z 24 480-499 13 13 13 22 22 22 F F F
17242024-02-03T17:23:18.171Z 25 500-519 15 15 15 24 24 24 F F F
17252024-02-03T17:23:18.171Z 26 520-539 10 10 10 17 17 17 F F F
17262024-02-03T17:23:18.171Z 27 540-559 15 15 15 24 24 24 F F F
17272024-02-03T17:23:18.172Z 28 560-579 10 10 10 17 17 17 F F F
17282024-02-03T17:23:18.172Z 29 580-599 15 15 15 24 24 24 F F F
17292024-02-03T17:23:18.172ZMax gen: 15, Max flush: 24
17302024-02-03T17:23:18.172Z
17312024-02-03T17:23:18.172ZBegin loop 8
17322024-02-03T17:23:18.172ZDownstairs to restart: 1
17332024-02-03T17:23:18.213Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.357932817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193}
17342024-02-03T17:23:18.213Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.358433881Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193}
17352024-02-03T17:23:18.213Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.358462321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193}
17362024-02-03T17:23:18.220Z{"msg":"Crucible c330554c-cfbf-4caf-b353-85aa608eea13 has session id: 35375ab7-a25a-4b80-9ef0-15e1fc27a937","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.358494633Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17372024-02-03T17:23:18.220Z{"msg":"Upstairs opts: Upstairs UUID: c330554c-cfbf-4caf-b353-85aa608eea13, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.358587846Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17382024-02-03T17:23:18.220Z{"msg":"Crucible stats registered with UUID: c330554c-cfbf-4caf-b353-85aa608eea13","v":Crucible runtime is spawned
17392024-02-03T17:23:18.220Z0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.358729804Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17402024-02-03T17:23:18.220Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.358809977Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193}
17412024-02-03T17:23:18.220Z{"msg":"c330554c-cfbf-4caf-b353-85aa608eea13 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359112302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17422024-02-03T17:23:18.220Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359188344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17432024-02-03T17:23:18.221Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.35923567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17442024-02-03T17:23:18.221Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359270231Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17452024-02-03T17:23:18.221Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359311897Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17462024-02-03T17:23:18.221Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359354729Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17472024-02-03T17:23:18.221Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.35952416Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17482024-02-03T17:23:18.221Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359614238Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17492024-02-03T17:23:18.221Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359687461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17502024-02-03T17:23:18.221Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359762991Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17512024-02-03T17:23:18.221Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359805879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17522024-02-03T17:23:18.221Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359847094Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17532024-02-03T17:23:18.221Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.359919622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17542024-02-03T17:23:18.221Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.3610342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17552024-02-03T17:23:18.222Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.361159696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17562024-02-03T17:23:18.222Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.361333646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17572024-02-03T17:23:18.222Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.361394968Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17582024-02-03T17:23:18.222Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.361931227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17592024-02-03T17:23:18.222Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.362151772Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17602024-02-03T17:23:18.222Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.362486813Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17612024-02-03T17:23:18.222Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.362553768Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17622024-02-03T17:23:18.222Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.362905414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17632024-02-03T17:23:18.222Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.363082344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17642024-02-03T17:23:18.222Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.363135279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17652024-02-03T17:23:18.222Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.363159337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17662024-02-03T17:23:18.223Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.363216496Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17672024-02-03T17:23:18.223Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.363240817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17682024-02-03T17:23:18.223Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.363435559Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17692024-02-03T17:23:18.223Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.36417529Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17702024-02-03T17:23:18.223Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364230054Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17712024-02-03T17:23:18.223Z{"msg":"[0]R flush_numbers[0..12]: [10, 21, 21, 17, 19, 24, 23, 20, 22, 22, 22, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.36430064Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17722024-02-03T17:23:18.223Z{"msg":"[0]R generation[0..12]: [6, 12, 12, 10, 10, 15, 14, 11, 13, 13, 13, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364330286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17732024-02-03T17:23:18.223Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364374617Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17742024-02-03T17:23:18.223Z{"msg":"[1]R flush_numbers[0..12]: [10, 21, 21, 17, 19, 24, 23, 20, 22, 22, 22, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.36439917Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17752024-02-03T17:23:18.223Z{"msg":"[1]R generation[0..12]: [6, 12, 12, 10, 10, 15, 14, 11, 13, 13, 13, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364443203Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17762024-02-03T17:23:18.223Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364501428Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17772024-02-03T17:23:18.224Z{"msg":"[2]R flush_numbers[0..12]: [10, 21, 21, 17, 19, 24, 23, 20, 22, 22, 22, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364528067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17782024-02-03T17:23:18.224Z{"msg":"[2]R generation[0..12]: [6, 12, 12, 10, 10, 15, 14, 11, 13, 13, 13, 15]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
17792024-02-03T17:23:18.224Z,"level":30,"time":"2024-02-03T17:23:17.36455542Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17802024-02-03T17:23:18.224Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364579374Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17812024-02-03T17:23:18.224Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.36462586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17822024-02-03T17:23:18.224Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364650127Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17832024-02-03T17:23:18.224Z{"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364715435Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17842024-02-03T17:23:18.224Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.36474019Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17852024-02-03T17:23:18.224Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364762521Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17862024-02-03T17:23:18.224Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364801577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17872024-02-03T17:23:18.224Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.3648251Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17882024-02-03T17:23:18.225Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364846655Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17892024-02-03T17:23:18.225Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364868172Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17902024-02-03T17:23:18.225Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.36491288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17912024-02-03T17:23:18.225Z{"msg":"c330554c-cfbf-4caf-b353-85aa608eea13 is now active with session: 35375ab7-a25a-4b80-9ef0-15e1fc27a937","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364936716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17922024-02-03T17:23:18.225Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.364956937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193,"session_id":"35375ab7-a25a-4b80-9ef0-15e1fc27a937"}
17932024-02-03T17:23:18.225Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:17.365000642Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193}
17942024-02-03T17:23:18.225ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
17952024-02-03T17:23:18.225ZLoading write count information from file "/tmp/test_repair_verify.data"
17962024-02-03T17:23:18.225ZRun Repair workload
17972024-02-03T17:23:18.225Z01/30 Write block 554 len 32768 data: 4 4 5 4 4 4 3 2
17982024-02-03T17:23:18.225Z02/30 Flush
17992024-02-03T17:23:18.226Z03/30 Write block 330 len 20480 data: 2 2 2 2 2
18002024-02-03T17:23:18.226Z04/30 Read block 486 len 28672
18012024-02-03T17:23:18.230Z05/30 Write block 338 len 40960 data: 2 3 3 3 3 3 3 4 4 4
18022024-02-03T17:23:18.230Z06/30 Read block 98 len 36864
18032024-02-03T17:23:18.230Z07/30 Read block 555 len 32768
18042024-02-03T17:23:18.230Z08/30 Write block 505 len 16384 data: 3 4 4 5
18052024-02-03T17:23:18.230Z09/30 Read block 282 len 24576
18062024-02-03T17:23:18.230Z10/30 Read block 540 len 4096
18072024-02-03T17:23:18.230Z11/30 Write block 210 len 24576 data: 3 3 3 3 3 3
18082024-02-03T17:23:18.230Z12/30 Flush
18092024-02-03T17:23:18.234Z13/30 Write block 148 len 28672 data: 2 2 2 2 2 2 3
18102024-02-03T17:23:18.235Z14/30 Read block 456 len 4096
18112024-02-03T17:23:18.239Z15/30 Read block 70 len 40960
18122024-02-03T17:23:18.239Z16/30 Write block 123 len 16384 data: 3 2 2 2
18132024-02-03T17:23:18.239Z17/30 Write block 149 len 36864 data: 3 3 3 3 3 4 3 3 3
18142024-02-03T17:23:18.239Z18/30 Read block 139 len 40960
18152024-02-03T17:23:18.243Z19/30 Write block 439 len 12288 data: 2 4 4
18162024-02-03T17:23:18.243Z20/30 Read block 107 len 28672
18172024-02-03T17:23:18.248Z21/30 Write block 590 len 40960 data: 3 4 4 4 4 4 4 4 4 3
18182024-02-03T17:23:18.248Z22/30 Read block 372 len 20480
18192024-02-03T17:23:18.248Z23/30 Write block 250 len 28672 data: 2 2 2 2 3 3 3
18202024-02-03T17:23:18.252Z24/30 Write block 150 len 8192 data: 4 4
18212024-02-03T17:23:18.252Z25/30 Read block 500 len 24576
18222024-02-03T17:23:18.256Z26/30 Flush
18232024-02-03T17:23:18.260Z27/30 Write block 93 len 16384 data: 2 2 2 2
18242024-02-03T17:23:18.260Z28/30 Read block 234 len 36864
18252024-02-03T17:23:18.263Z29/30 Write block 1 len 36864 data: 2 2 2 3 3 3 3 4 3
18262024-02-03T17:23:18.263Z30/30 Read block 82 len 36864
18272024-02-03T17:23:18.268Z----------------------------------------------------------------
18282024-02-03T17:23:18.268Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18292024-02-03T17:23:18.268ZStates: Active Active Active
18302024-02-03T17:23:18.268ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18312024-02-03T17:23:18.268Z 1 Acked 1000 Write 8 Done Sent Done false
18322024-02-03T17:23:18.268Z 2 Acked 1001 Flush 0 Done Sent Done false
18332024-02-03T17:23:18.268Z 3 Acked 1002 Write 5 Done Sent Done false
18342024-02-03T17:23:18.268Z 4 Acked 1003 Read 7 Done Sent Done false
18352024-02-03T17:23:18.268Z 5 Acked 1004 Write 10 Done Sent Done false
18362024-02-03T17:23:18.269Z 6 Acked 1005 Read 9 Done Sent Done false
18372024-02-03T17:23:18.269Z 7 Acked 1006 Read 8 Done Sent Done false
18382024-02-03T17:23:18.269Z 8 Acked 1007 Write 4 Done Sent Done false
18392024-02-03T17:23:18.269Z 9 Acked 1008 Read 6 Done Sent Done false
18402024-02-03T17:23:18.269Z 10 Acked 1009 Read 1 Done Sent Done false
18412024-02-03T17:23:18.269Z 11 Acked 1010 Write 6 Done Sent Done false
18422024-02-03T17:23:18.269Z 12 Acked 1011 Flush 0 Done Sent Done false
18432024-02-03T17:23:18.269Z 13 Acked 1012 Write 7 Done Sent Done false
18442024-02-03T17:23:18.269Z 14 Acked 1013 Read 1 Done Sent Done false
18452024-02-03T17:23:18.269Z 15 Acked 1014 Read 10 Done Sent Done false
18462024-02-03T17:23:18.269Z 16 Acked 1015 Write 4 Done Sent Done false
18472024-02-03T17:23:18.269Z 17 Acked 1016 Write 9 Done Sent Done false
18482024-02-03T17:23:18.269Z 18 Acked 1017 Read 10 Done Sent Done false
18492024-02-03T17:23:18.270Z 19 Acked 1018 Write 3 Done Sent Done false
18502024-02-03T17:23:18.270Z 20 Acked 1019 Read 7 Done Sent Done false
18512024-02-03T17:23:18.270Z 21 Acked 1020 Write 10 Done Sent Done false
18522024-02-03T17:23:18.270Z 22 Acked 1021 Read 5 Done Sent Done false
18532024-02-03T17:23:18.270Z 23 Acked 1022 Write 7 Done Sent Done false
18542024-02-03T17:23:18.270Z 24 Acked 1023 Write 2 Done Sent Done false
18552024-02-03T17:23:18.270Z 25 Acked 1024 Read 6 Done Sent Done false
18562024-02-03T17:23:18.270Z 26 Acked 1025 Flush 0 Done Sent Done false
18572024-02-03T17:23:18.270Z 27 Acked 1026 Write 4 Done Sent Done false
18582024-02-03T17:23:18.270Z 28 Acked 1027 Read 9 Done Sent Done false
18592024-02-03T17:23:18.270Z 29 Acked 1028 Write 9 Done Sent Done false
18602024-02-03T17:23:18.270Z 30 Acked 1029 Read 9 Done Sent Sent false
18612024-02-03T17:23:18.271Z STATES DS:0 DS:1 DS:2 TOTAL
18622024-02-03T17:23:18.271Z New 0 0 0 0
18632024-02-03T17:23:18.271Z Sent 0 30 1 31
18642024-02-03T17:23:18.271Z Done 30 0 29 59
18652024-02-03T17:23:18.271Z Skipped 0 0 0 0
18662024-02-03T17:23:18.271Z Error 0 0 0 0
18672024-02-03T17:23:18.271ZLast Flush: 1025 0 1025
18682024-02-03T17:23:18.271ZDownstairs last five completed:
18692024-02-03T17:23:18.271ZUpstairs last five completed: 30 29 28 27 26
18702024-02-03T17:23:18.271Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:17.421531226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1193}
18712024-02-03T17:23:18.271ZWrote out file "/tmp/test_repair_verify.data"
18722024-02-03T17:23:18.275Z
18732024-02-03T17:23:18.280ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18742024-02-03T17:23:18.304ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18752024-02-03T17:23:18.304Z 0 000-019 6 6 6 10 10 10 T F T <---
18762024-02-03T17:23:18.308Z 1 020-039 12 12 12 21 21 21 F F F
18772024-02-03T17:23:18.308Z 2 040-059 12 12 12 21 21 21 F F F
18782024-02-03T17:23:18.308Z 3 060-079 10 10 10 17 17 17 F F F
18792024-02-03T17:23:18.309Z 4 080-099 10 10 10 19 19 19 T F T <---
18802024-02-03T17:23:18.309Z 5 100-119 15 15 15 24 24 24 F F F
18812024-02-03T17:23:18.309Z 6 120-139 16 14 16 27 23 27 F F F <---
18822024-02-03T17:23:18.309Z 7 140-159 16 11 16 27 20 27 F F F <---
18832024-02-03T17:23:18.309Z 8 160-179 13 13 13 22 22 22 F F F
18842024-02-03T17:23:18.309Z 9 180-199 13 13 13 22 22 22 F F F
18852024-02-03T17:23:18.309Z 10 200-219 16 13 16 26 22 26 F F F <---
18862024-02-03T17:23:18.309Z 11 220-239 15 15 15 24 24 24 F F F
18872024-02-03T17:23:18.309Z 12 240-259 16 15 16 27 24 27 F F F <---
18882024-02-03T17:23:18.309Z 13 260-279 15 15 15 24 24 24 F F F
18892024-02-03T17:23:18.309Z 14 280-299 6 6 6 8 8 8 F F F
18902024-02-03T17:23:18.309Z 15 300-319 14 14 14 23 23 23 F F F
18912024-02-03T17:23:18.309Z 16 320-339 16 10 16 26 17 26 F F F <---
18922024-02-03T17:23:18.310Z 17 340-359 16 15 16 26 24 26 F F F <---
18932024-02-03T17:23:18.310Z 18 360-379 15 15 15 24 24 24 F F F
18942024-02-03T17:23:18.310Z 19 380-399 13 13 13 22 22 22 F F F
18952024-02-03T17:23:18.310Z 20 400-419 11 11 11 20 20 20 F F F
18962024-02-03T17:23:18.310Z 21 420-439 16 9 16 27 16 27 F F F <---
18972024-02-03T17:23:18.310Z 22 440-459 16 13 16 27 22 27 F F F <---
18982024-02-03T17:23:18.310Z 23 460-479 12 12 12 21 21 21 F F F
18992024-02-03T17:23:18.310Z 24 480-499 13 13 13 22 22 22 F F F
19002024-02-03T17:23:18.310Z 25 500-519 16 15 16 26 24 26 F F F <---
19012024-02-03T17:23:18.310Z 26 520-539 10 10 10 17 17 17 F F F
19022024-02-03T17:23:18.310Z 27 540-559 16 15 16 25 24 25 F F F <---
19032024-02-03T17:23:18.310Z 28 560-579 16 10 16 25 17 25 F F F <---
19042024-02-03T17:23:18.311Z 29 580-599 16 15 16 27 24 27 F F F <---
19052024-02-03T17:23:18.311ZMax gen: 16, Max flush: 27
19062024-02-03T17:23:18.311ZError: Difference in extent metadata found!
19072024-02-03T17:23:18.315ZA Difference in extent metadata is expected here
19082024-02-03T17:23:18.315ZOn loop 8
19092024-02-03T17:23:18.315Z
19102024-02-03T17:23:18.315ZVerifying data now
19112024-02-03T17:23:23.523ZNo change after verify
19122024-02-03T17:23:23.523ZLoop: 8 Downstairs dump after verify (and repair):
19132024-02-03T17:23:23.548ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19142024-02-03T17:23:23.548Z 0 000-019 17 17 17 28 28 28 F F F
19152024-02-03T17:23:23.548Z 1 020-039 12 12 12 21 21 21 F F F
19162024-02-03T17:23:23.551Z 2 040-059 12 12 12 21 21 21 F F F
19172024-02-03T17:23:23.551Z 3 060-079 10 10 10 17 17 17 F F F
19182024-02-03T17:23:23.551Z 4 080-099 17 17 17 28 28 28 F F F
19192024-02-03T17:23:23.551Z 5 100-119 15 15 15 24 24 24 F F F
19202024-02-03T17:23:23.551Z 6 120-139 16 16 16 27 27 27 F F F
19212024-02-03T17:23:23.551Z 7 140-159 16 16 16 27 27 27 F F F
19222024-02-03T17:23:23.551Z 8 160-179 13 13 13 22 22 22 F F F
19232024-02-03T17:23:23.551Z 9 180-199 13 13 13 22 22 22 F F F
19242024-02-03T17:23:23.551Z 10 200-219 16 16 16 26 26 26 F F F
19252024-02-03T17:23:23.551Z 11 220-239 15 15 15 24 24 24 F F F
19262024-02-03T17:23:23.552Z 12 240-259 16 16 16 27 27 27 F F F
19272024-02-03T17:23:23.552Z 13 260-279 15 15 15 24 24 24 F F F
19282024-02-03T17:23:23.552Z 14 280-299 6 6 6 8 8 8 F F F
19292024-02-03T17:23:23.552Z 15 300-319 14 14 14 23 23 23 F F F
19302024-02-03T17:23:23.552Z 16 320-339 16 16 16 26 26 26 F F F
19312024-02-03T17:23:23.552Z 17 340-359 16 16 16 26 26 26 F F F
19322024-02-03T17:23:23.552Z 18 360-379 15 15 15 24 24 24 F F F
19332024-02-03T17:23:23.552Z 19 380-399 13 13 13 22 22 22 F F F
19342024-02-03T17:23:23.552Z 20 400-419 11 11 11 20 20 20 F F F
19352024-02-03T17:23:23.552Z 21 420-439 16 16 16 27 27 27 F F F
19362024-02-03T17:23:23.552Z 22 440-459 16 16 16 27 27 27 F F F
19372024-02-03T17:23:23.552Z 23 460-479 12 12 12 21 21 21 F F F
19382024-02-03T17:23:23.552Z 24 480-499 13 13 13 22 22 22 F F F
19392024-02-03T17:23:23.552Z 25 500-519 16 16 16 26 26 26 F F F
19402024-02-03T17:23:23.552Z 26 520-539 10 10 10 17 17 17 F F F
19412024-02-03T17:23:23.552Z 27 540-559 16 16 16 25 25 25 F F F
19422024-02-03T17:23:23.552Z 28 560-579 16 16 16 25 25 25 F F F
19432024-02-03T17:23:23.552Z 29 580-599 16 16 16 27 27 27 F F F
19442024-02-03T17:23:23.552ZMax gen: 17, Max flush: 28
19452024-02-03T17:23:23.555Z
19462024-02-03T17:23:23.556ZBegin loop 9
19472024-02-03T17:23:23.556ZDownstairs to restart: 2
19482024-02-03T17:23:23.585Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.742102236Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201}
19492024-02-03T17:23:23.586Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.742577503Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201}
19502024-02-03T17:23:23.598Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.742622087Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201}
19512024-02-03T17:23:23.598Z{"msg":"Crucible 68178d28-f242-4e83-a534-f06745141254 has session id: f74b985b-b68d-40e9-b51c-fb55ce43d2ce","v":0,"name":"crucible","level":30Crucible runtime is spawned
19522024-02-03T17:23:23.599Z,"time":"2024-02-03T17:23:22.74270701Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19532024-02-03T17:23:23.599Z{"msg":"Upstairs opts: Upstairs UUID: 68178d28-f242-4e83-a534-f06745141254, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.742868602Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19542024-02-03T17:23:23.599Z{"msg":"Crucible stats registered with UUID: 68178d28-f242-4e83-a534-f06745141254","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.742898166Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19552024-02-03T17:23:23.599Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.742963744Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201}
19562024-02-03T17:23:23.599Z{"msg":"68178d28-f242-4e83-a534-f06745141254 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743057207Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19572024-02-03T17:23:23.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743120381Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19582024-02-03T17:23:23.599Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743149127Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19592024-02-03T17:23:23.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743206147Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19602024-02-03T17:23:23.599Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743230977Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19612024-02-03T17:23:23.599Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743253218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19622024-02-03T17:23:23.599Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743288492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19632024-02-03T17:23:23.599Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743341714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19642024-02-03T17:23:23.600Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743410046Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19652024-02-03T17:23:23.600Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743456523Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19662024-02-03T17:23:23.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743757662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19672024-02-03T17:23:23.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743844313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19682024-02-03T17:23:23.600Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.743884358Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19692024-02-03T17:23:23.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.745285165Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19702024-02-03T17:23:23.600Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.745380692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19712024-02-03T17:23:23.600Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.745427073Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19722024-02-03T17:23:23.601Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.745468057Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19732024-02-03T17:23:23.601Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.746236305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19742024-02-03T17:23:23.606Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.746460089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19752024-02-03T17:23:23.606Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.746515084Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19762024-02-03T17:23:23.606Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.746591014Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19772024-02-03T17:23:23.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.747076796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19782024-02-03T17:23:23.606Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.747147158Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19792024-02-03T17:23:23.606Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.747192778Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19802024-02-03T17:23:23.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.747422268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19812024-02-03T17:23:23.606Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.747489001Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19822024-02-03T17:23:23.606Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.747532876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19832024-02-03T17:23:23.606Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.747651719Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19842024-02-03T17:23:23.607Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748334072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19852024-02-03T17:23:23.607Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748411646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19862024-02-03T17:23:23.607Z{"msg":"[0]R flush_numbers[0..12]: [28, 21, 21, 17, 28, 24, 27, 27, 22, 22, 26, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748439309Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19872024-02-03T17:23:23.607Z{"msg":"[0]R generation[0..12]: [17, 12, 12, 10, 17, 15, 16, 16, 13, 13, 16, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748462797Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19882024-02-03T17:23:23.607Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748487053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19892024-02-03T17:23:23.607Z{"msg":"[1]R flush_numbers[0..12]: [28, 21, 21, 17, 28, 24, 27, 27, 22, 22, 26, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748529727Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19902024-02-03T17:23:23.607Z{"msg":"[1]R generation[0..12]: [17, 12, 12, 10, 17, 15, 16, 16, 13, 13, 16, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748554913Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19912024-02-03T17:23:23.607Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748579313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19922024-02-03T17:23:23.607Z{"msg":"[2]R flush_numbers[0..12]: [28, 21, 21, 17, 28, 24, 27, 27, 22, 22, 26, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.74860299Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19932024-02-03T17:23:23.607Z{"msg":"[2]R generation[0..12]: [17, 12, 12, 10, 17, 15, 16, 16, 13, 13, 16, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748626189Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19942024-02-03T17:23:23.607Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748668281Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19952024-02-03T17:23:23.607Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748692529Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19962024-02-03T17:23:23.607Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748714893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19972024-02-03T17:23:23.607ZWait for a query_work_queue command to finish before sending IO
19982024-02-03T17:23:23.607Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748759696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
19992024-02-03T17:23:23.607Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748783404Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20002024-02-03T17:23:23.607Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748808924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20012024-02-03T17:23:23.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748868321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20022024-02-03T17:23:23.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.74889585Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20032024-02-03T17:23:23.607Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748938323Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20042024-02-03T17:23:23.607Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.748966895Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20052024-02-03T17:23:23.607Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.749027016Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20062024-02-03T17:23:23.607Z{"msg":"68178d28-f242-4e83-a534-f06745141254 is now active with session: f74b985b-b68d-40e9-b51c-fb55ce43d2ce","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.74906313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20072024-02-03T17:23:23.608Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.749094124Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201,"session_id":"f74b985b-b68d-40e9-b51c-fb55ce43d2ce"}
20082024-02-03T17:23:23.608Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:22.749119209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201}
20092024-02-03T17:23:23.608ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20102024-02-03T17:23:23.608ZLoading write count information from file "/tmp/test_repair_verify.data"
20112024-02-03T17:23:23.608ZRun Repair workload
20122024-02-03T17:23:23.608Z01/30 Write block 328 len 4096 data: 2
20132024-02-03T17:23:23.608Z02/30 Read block 304 len 8192
20142024-02-03T17:23:23.608Z03/30 Read block 463 len 32768
20152024-02-03T17:23:23.608Z04/30 Read block 264 len 16384
20162024-02-03T17:23:23.608Z05/30 Write block 114 len 16384 data: 4 3 3 5
20172024-02-03T17:23:23.608Z06/30 Read block 284 len 28672
20182024-02-03T17:23:23.608Z07/30 Write block 333 len 16384 data: 3 3 2 2
20192024-02-03T17:23:23.608Z08/30 Read block 40 len 20480
20202024-02-03T17:23:23.608Z09/30 Flush
20212024-02-03T17:23:23.611Z10/30 Write block 11 len 16384 data: 2 2 2 2
20222024-02-03T17:23:23.611Z11/30 Read block 11 len 36864
20232024-02-03T17:23:23.611Z12/30 Read block 482 len 16384
20242024-02-03T17:23:23.614Z13/30 Write block 394 len 40960 data: 4 5 5 4 5 5 4 3 3 3
20252024-02-03T17:23:23.614Z14/30 Read block 373 len 32768
20262024-02-03T17:23:23.617Z15/30 Write block 125 len 36864 data: 3 3 2 2 2 3 3 3 3
20272024-02-03T17:23:23.621Z16/30 Write block 414 len 4096 data: 2
20282024-02-03T17:23:23.621Z17/30 Read block 473 len 36864
20292024-02-03T17:23:23.621Z18/30 Write block 337 len 16384 data: 2 3 4 4
20302024-02-03T17:23:23.624Z19/30 Flush
20312024-02-03T17:23:23.627Z20/30 Write block 391 len 20480 data: 2 2 4 5 6
20322024-02-03T17:23:23.627Z21/30 Write block 356 len 40960 data: 2 2 2 2 3 3 4 4 4 5
20332024-02-03T17:23:23.634Z22/30 Write block 477 len 12288 data: 4 4 4
20342024-02-03T17:23:23.634Z23/30 Read block 41 len 24576
20352024-02-03T17:23:23.634Z24/30 Read block 457 len 24576
20362024-02-03T17:23:23.637Z25/30 Read block 432 len 4096
20372024-02-03T17:23:23.637Z26/30 Write block 181 len 8192 data: 3 3
20382024-02-03T17:23:23.637Z27/30 Read block 144 len 8192
20392024-02-03T17:23:23.637Z28/30 Read block 481 len 4096
20402024-02-03T17:23:23.637Z29/30 Write block 290 len 32768 data: 2 2 3 3 2 2 2 2
20412024-02-03T17:23:23.641Z30/30 Write block 460 len 24576 data: 3 3 3 3 3 4
20422024-02-03T17:23:23.641Z----------------------------------------------------------------
20432024-02-03T17:23:23.641Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20442024-02-03T17:23:23.641ZStates: Active Active Active
20452024-02-03T17:23:23.641ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20462024-02-03T17:23:23.641Z 1 Acked 1000 Write 1 Done Done Sent false
20472024-02-03T17:23:23.642Z 2 Acked 1001 Read 2 Done Done Sent false
20482024-02-03T17:23:23.642Z 3 Acked 1002 Read 8 Done Done Sent false
20492024-02-03T17:23:23.642Z 4 Acked 1003 Read 4 Done Done Sent false
20502024-02-03T17:23:23.642Z 5 Acked 1004 Write 4 Done Done Sent false
20512024-02-03T17:23:23.642Z 6 Acked 1005 Read 7 Done Done Sent false
20522024-02-03T17:23:23.642Z 7 Acked 1006 Write 4 Done Done Sent false
20532024-02-03T17:23:23.642Z 8 Acked 1007 Read 5 Done Done Sent false
20542024-02-03T17:23:23.642Z 9 Acked 1008 Flush 0 Done Done Sent false
20552024-02-03T17:23:23.642Z 10 Acked 1009 Write 4 Done Done Sent false
20562024-02-03T17:23:23.642Z 11 Acked 1010 Read 9 Done Done Sent false
20572024-02-03T17:23:23.642Z 12 Acked 1011 Read 4 Done Done Sent false
20582024-02-03T17:23:23.642Z 13 Acked 1012 Write 10 Done Done Sent false
20592024-02-03T17:23:23.642Z 14 Acked 1013 Read 8 Done Done Sent false
20602024-02-03T17:23:23.642Z 15 Acked 1014 Write 9 Done Done Sent false
20612024-02-03T17:23:23.642Z 16 Acked 1015 Write 1 Done Done Sent false
20622024-02-03T17:23:23.642Z 17 Acked 1016 Read 9 Done Done Sent false
20632024-02-03T17:23:23.642Z 18 Acked 1017 Write 4 Done Done Sent false
20642024-02-03T17:23:23.642Z 19 Acked 1018 Flush 0 Done Done Sent false
20652024-02-03T17:23:23.642Z 20 Acked 1019 Write 5 Done Done Sent false
20662024-02-03T17:23:23.642Z 21 Acked 1020 Write 10 Done Done Sent false
20672024-02-03T17:23:23.642Z 22 Acked 1021 Write 3 Done Done Sent false
20682024-02-03T17:23:23.642Z 23 Acked 1022 Read 6 Done Done Sent false
20692024-02-03T17:23:23.642Z 24 Acked 1023 Read 6 Done Done Sent false
20702024-02-03T17:23:23.642Z 25 Acked 1024 Read 1 Done Done Sent false
20712024-02-03T17:23:23.643Z 26 Acked 1025 Write 2 Done Done Sent false
20722024-02-03T17:23:23.643Z 27 Acked 1026 Read 2 Done Done Sent false
20732024-02-03T17:23:23.643Z 28 Acked 1027 Read 1 Done Done Sent false
20742024-02-03T17:23:23.643Z 29 Acked 1028 Write 8 Done Done Sent false
20752024-02-03T17:23:23.643Z 30 Acked 1029 Write 6 Sent Sent Sent false
20762024-02-03T17:23:23.643Z STATES DS:0 DS:1 DS:2 TOTAL
20772024-02-03T17:23:23.643Z New 0 0 0 0
20782024-02-03T17:23:23.643Z Sent 1 1 30 32
20792024-02-03T17:23:23.643Z Done 29 29 0 58
20802024-02-03T17:23:23.643Z Skipped 0 0 0 0
20812024-02-03T17:23:23.643Z Error 0 0 0 0
20822024-02-03T17:23:23.643ZLast Flush: 1018 1018 0
20832024-02-03T17:23:23.643ZDownstairs last five completed:
20842024-02-03T17:23:23.643ZUpstairs last five completed: 30 29 28 27 26
20852024-02-03T17:23:23.643Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:22.796024372Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1201}
20862024-02-03T17:23:23.643ZWrote out file "/tmp/test_repair_verify.data"
20872024-02-03T17:23:23.647Z
20882024-02-03T17:23:23.654ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20892024-02-03T17:23:23.682ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20902024-02-03T17:23:23.682Z 0 000-019 18 18 17 30 30 28 F F F <---
20912024-02-03T17:23:23.682Z 1 020-039 12 12 12 21 21 21 F F F
20922024-02-03T17:23:23.682Z 2 040-059 12 12 12 21 21 21 F F F
20932024-02-03T17:23:23.682Z 3 060-079 10 10 10 17 17 17 F F F
20942024-02-03T17:23:23.682Z 4 080-099 17 17 17 28 28 28 F F F
20952024-02-03T17:23:23.682Z 5 100-119 18 18 15 29 29 24 F F F <---
20962024-02-03T17:23:23.682Z 6 120-139 18 18 16 30 30 27 F F F <---
20972024-02-03T17:23:23.682Z 7 140-159 16 16 16 27 27 27 F F F
20982024-02-03T17:23:23.686Z 8 160-179 13 13 13 22 22 22 F F F
20992024-02-03T17:23:23.686Z 9 180-199 13 13 13 22 22 22 T T F <---
21002024-02-03T17:23:23.686Z 10 200-219 16 16 16 26 26 26 F F F
21012024-02-03T17:23:23.686Z 11 220-239 15 15 15 24 24 24 F F F
21022024-02-03T17:23:23.686Z 12 240-259 16 16 16 27 27 27 F F F
21032024-02-03T17:23:23.686Z 13 260-279 15 15 15 24 24 24 F F F
21042024-02-03T17:23:23.686Z 14 280-299 6 6 6 8 8 8 T T F <---
21052024-02-03T17:23:23.686Z 15 300-319 14 14 14 23 23 23 F F F
21062024-02-03T17:23:23.686Z 16 320-339 18 18 16 30 30 26 F F F <---
21072024-02-03T17:23:23.686Z 17 340-359 18 18 16 30 30 26 T T F <---
21082024-02-03T17:23:23.686Z 18 360-379 15 15 15 24 24 24 T T F <---
21092024-02-03T17:23:23.686Z 19 380-399 18 18 13 30 30 22 T T F <---
21102024-02-03T17:23:23.687Z 20 400-419 18 18 11 30 30 20 F F F <---
21112024-02-03T17:23:23.687Z 21 420-439 16 16 16 27 27 27 F F F
21122024-02-03T17:23:23.687Z 22 440-459 16 16 16 27 27 27 F F F
21132024-02-03T17:23:23.687Z 23 460-479 12 12 12 21 21 21 T T F <---
21142024-02-03T17:23:23.687Z 24 480-499 13 13 13 22 22 22 F F F
21152024-02-03T17:23:23.687Z 25 500-519 16 16 16 26 26 26 F F F
21162024-02-03T17:23:23.687Z 26 520-539 10 10 10 17 17 17 F F F
21172024-02-03T17:23:23.687Z 27 540-559 16 16 16 25 25 25 F F F
21182024-02-03T17:23:23.687Z 28 560-579 16 16 16 25 25 25 F F F
21192024-02-03T17:23:23.687Z 29 580-599 16 16 16 27 27 27 F F F
21202024-02-03T17:23:23.687ZMax gen: 18, Max flush: 30
21212024-02-03T17:23:23.687ZError: Difference in extent metadata found!
21222024-02-03T17:23:23.692ZA Difference in extent metadata is expected here
21232024-02-03T17:23:23.692ZOn loop 9
21242024-02-03T17:23:23.692Z
21252024-02-03T17:23:23.692ZVerifying data now
21262024-02-03T17:23:28.818ZNo change after verify
21272024-02-03T17:23:28.818ZLoop: 9 Downstairs dump after verify (and repair):
21282024-02-03T17:23:28.825ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21292024-02-03T17:23:28.825Z 0 000-019 18 18 18 30 30 30 F F F
21302024-02-03T17:23:28.825Z 1 020-039 12 12 12 21 21 21 F F F
21312024-02-03T17:23:28.826Z 2 040-059 12 12 12 21 21 21 F F F
21322024-02-03T17:23:28.826Z 3 060-079 10 10 10 17 17 17 F F F
21332024-02-03T17:23:28.826Z 4 080-099 17 17 17 28 28 28 F F F
21342024-02-03T17:23:28.826Z 5 100-119 18 18 18 29 29 29 F F F
21352024-02-03T17:23:28.826Z 6 120-139 18 18 18 30 30 30 F F F
21362024-02-03T17:23:28.826Z 7 140-159 16 16 16 27 27 27 F F F
21372024-02-03T17:23:28.826Z 8 160-179 13 13 13 22 22 22 F F F
21382024-02-03T17:23:28.826Z 9 180-199 19 19 19 31 31 31 F F F
21392024-02-03T17:23:28.826Z 10 200-219 16 16 16 26 26 26 F F F
21402024-02-03T17:23:28.826Z 11 220-239 15 15 15 24 24 24 F F F
21412024-02-03T17:23:28.826Z 12 240-259 16 16 16 27 27 27 F F F
21422024-02-03T17:23:28.826Z 13 260-279 15 15 15 24 24 24 F F F
21432024-02-03T17:23:28.826Z 14 280-299 19 19 19 31 31 31 F F F
21442024-02-03T17:23:28.826Z 15 300-319 14 14 14 23 23 23 F F F
21452024-02-03T17:23:28.827Z 16 320-339 18 18 18 30 30 30 F F F
21462024-02-03T17:23:28.827Z 17 340-359 19 19 19 31 31 31 F F F
21472024-02-03T17:23:28.827Z 18 360-379 19 19 19 31 31 31 F F F
21482024-02-03T17:23:28.827Z 19 380-399 19 19 19 31 31 31 F F F
21492024-02-03T17:23:28.827Z 20 400-419 18 18 18 30 30 30 F F F
21502024-02-03T17:23:28.827Z 21 420-439 16 16 16 27 27 27 F F F
21512024-02-03T17:23:28.827Z 22 440-459 16 16 16 27 27 27 F F F
21522024-02-03T17:23:28.827Z 23 460-479 19 19 19 31 31 31 F F F
21532024-02-03T17:23:28.827Z 24 480-499 13 13 13 22 22 22 F F F
21542024-02-03T17:23:28.827Z 25 500-519 16 16 16 26 26 26 F F F
21552024-02-03T17:23:28.827Z 26 520-539 10 10 10 17 17 17 F F F
21562024-02-03T17:23:28.827Z 27 540-559 16 16 16 25 25 25 F F F
21572024-02-03T17:23:28.827Z 28 560-579 16 16 16 25 25 25 F F F
21582024-02-03T17:23:28.827Z 29 580-599 16 16 16 27 27 27 F F F
21592024-02-03T17:23:28.828ZMax gen: 19, Max flush: 31
21602024-02-03T17:23:28.828Z
21612024-02-03T17:23:28.828ZBegin loop 10
21622024-02-03T17:23:28.828ZDownstairs to restart: 2
21632024-02-03T17:23:28.828Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.912430245Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209}
21642024-02-03T17:23:28.828Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.91304541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209}
21652024-02-03T17:23:28.828Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.91309551Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209}
21662024-02-03T17:23:28.828Z{"msg":"Crucible f8b50125-276a-4493-a302-5047b4d51e14 has session id: 0c37b0e6-17bb-473e-9884-22d6f3a58dd8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913134062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21672024-02-03T17:23:28.828Z{"msg":"Upstairs opts: Upstairs UUID: f8b50125-276a-4493-a302-5047b4d51e14, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913190785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21682024-02-03T17:23:28.828ZCrucible runtime is spawned
21692024-02-03T17:23:28.828Z{"msg":"Crucible stats registered with UUID: f8b50125-276a-4493-a302-5047b4d51e14","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913282348Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21702024-02-03T17:23:28.828Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913352775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209}
21712024-02-03T17:23:28.828Z{"msg":"f8b50125-276a-4493-a302-5047b4d51e14 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913465574Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21722024-02-03T17:23:28.829Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913533259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21732024-02-03T17:23:28.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913565097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21742024-02-03T17:23:28.829Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913633796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21752024-02-03T17:23:28.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913678177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21762024-02-03T17:23:28.829Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913729503Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21772024-02-03T17:23:28.829Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913754824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21782024-02-03T17:23:28.829Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913781558Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21792024-02-03T17:23:28.829Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913846907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21802024-02-03T17:23:28.829Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.91389076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21812024-02-03T17:23:28.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.913964072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21822024-02-03T17:23:28.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.914005795Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21832024-02-03T17:23:28.829Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.914179065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21842024-02-03T17:23:28.829Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.915405181Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21852024-02-03T17:23:28.830Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.915468263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21862024-02-03T17:23:28.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.91556596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21872024-02-03T17:23:28.830Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.9155972Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21882024-02-03T17:23:28.830Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.916432334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21892024-02-03T17:23:28.830Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.916586153Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21902024-02-03T17:23:28.830Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.916617539Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21912024-02-03T17:23:28.830Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.916657573Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21922024-02-03T17:23:28.830Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.917350028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21932024-02-03T17:23:28.830Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.917666245Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21942024-02-03T17:23:28.830Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.917743869Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21952024-02-03T17:23:28.830Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.917789196Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21962024-02-03T17:23:28.830Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.917829957Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21972024-02-03T17:23:28.830Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.917872049Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21982024-02-03T17:23:28.830Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.917931974Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
21992024-02-03T17:23:28.831Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918251642Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22002024-02-03T17:23:28.831Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918314058Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22012024-02-03T17:23:28.831Z{"msg":"[0]R flush_numbers[0..12]: [30, 21, 21, 17, 28, 29, 30, 27, 22, 31, 26, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918341643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22022024-02-03T17:23:28.831Z{"msg":"[0]R generation[0..12]: [18, 12, 12, 10, 17, 18, 18, 16, 13, 19, 16, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918384295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22032024-02-03T17:23:28.831Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.9184109Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22042024-02-03T17:23:28.831Z{"msg":"[1]R flush_numbers[0..12]: [30, 21, 21, 17, 28, 29, 30, 27, 22, 31, 26, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918434875Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22052024-02-03T17:23:28.831Z{"msg":"[1]R generation[0..12]: [18, 12, 12, 10, 17, 18, 18, 16, 13, 19, 16, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.91845838Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22062024-02-03T17:23:28.831Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918482415Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22072024-02-03T17:23:28.831Z{"msg":"[2]R flush_numbers[0..12]: [30, 21, 21, 17, 28, 29, 30, 27, 22, 31, 26, 24]",Wait for a query_work_queue command to finish before sending IO
22082024-02-03T17:23:28.831Z"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918549423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22092024-02-03T17:23:28.831Z{"msg":"[2]R generation[0..12]: [18, 12, 12, 10, 17, 18, 18, 16, 13, 19, 16, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918620685Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22102024-02-03T17:23:28.831Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918653745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22112024-02-03T17:23:28.831Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918715726Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22122024-02-03T17:23:28.831Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918748748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22132024-02-03T17:23:28.831Z{"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918821643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22142024-02-03T17:23:28.831Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918885022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22152024-02-03T17:23:28.832Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918916889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22162024-02-03T17:23:28.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.918974523Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22172024-02-03T17:23:28.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.919041022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22182024-02-03T17:23:28.832Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.919084243Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22192024-02-03T17:23:28.832Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.91914024Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22202024-02-03T17:23:28.832Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.919189854Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22212024-02-03T17:23:28.832Z{"msg":"f8b50125-276a-4493-a302-5047b4d51e14 is now active with session: 0c37b0e6-17bb-473e-9884-22d6f3a58dd8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.919289985Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22222024-02-03T17:23:28.832Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.91932846Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209,"session_id":"0c37b0e6-17bb-473e-9884-22d6f3a58dd8"}
22232024-02-03T17:23:28.832Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:27.919366226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209}
22242024-02-03T17:23:28.832ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22252024-02-03T17:23:28.832ZLoading write count information from file "/tmp/test_repair_verify.data"
22262024-02-03T17:23:28.832ZRun Repair workload
22272024-02-03T17:23:28.832Z01/30 Write block 548 len 28672 data: 2 2 2 4 5 5 5
22282024-02-03T17:23:28.837Z02/30 Write block 35 len 40960 data: 3 3 3 3 3 3 2 2 2 3
22292024-02-03T17:23:28.838Z03/30 Read block 447 len 4096
22302024-02-03T17:23:28.838Z04/30 Write block 325 len 36864 data: 2 2 2 3 2 3 3 3 4
22312024-02-03T17:23:28.838Z05/30 Read block 93 len 24576
22322024-02-03T17:23:28.838Z06/30 Read block 405 len 20480
22332024-02-03T17:23:28.838Z07/30 Read block 536 len 4096
22342024-02-03T17:23:28.838Z08/30 Read block 98 len 28672
22352024-02-03T17:23:28.838Z09/30 Read block 579 len 32768
22362024-02-03T17:23:28.838Z10/30 Read block 528 len 24576
22372024-02-03T17:23:28.838Z11/30 Flush
22382024-02-03T17:23:28.838Z12/30 Write block 167 len 20480 data: 3 3 4 4 3
22392024-02-03T17:23:28.838Z13/30 Read block 163 len 40960
22402024-02-03T17:23:28.838Z14/30 Write block 412 len 8192 data: 2 2
22412024-02-03T17:23:28.838Z15/30 Write block 443 len 16384 data: 4 4 3 3
22422024-02-03T17:23:28.839Z16/30 Read block 114 len 40960
22432024-02-03T17:23:28.839Z17/30 Write block 28 len 8192 data: 2 2
22442024-02-03T17:23:28.839Z18/30 Read block 460 len 8192
22452024-02-03T17:23:28.839Z19/30 Read block 517 len 4096
22462024-02-03T17:23:28.839Z20/30 Read block 343 len 28672
22472024-02-03T17:23:28.839Z21/30 Read block 591 len 24576
22482024-02-03T17:23:28.839Z22/30 Read block 382 len 8192
22492024-02-03T17:23:28.839Z23/30 Read block 115 len 28672
22502024-02-03T17:23:28.839Z24/30 Write block 206 len 8192 data: 2 2
22512024-02-03T17:23:28.839Z25/30 Write block 540 len 12288 data: 2 2 2
22522024-02-03T17:23:28.839Z26/30 Write block 525 len 28672 data: 2 3 4 3 4 4 3
22532024-02-03T17:23:28.839Z27/30 Write block 563 len 36864 data: 2 3 3 3 3 3 2 3 3
22542024-02-03T17:23:28.839Z28/30 Write block 402 len 40960 data: 4 4 2 2 2 2 2 2 2 2
22552024-02-03T17:23:28.840Z29/30 Write block 525 len 32768 data: 3 4 5 4 5 5 4 3
22562024-02-03T17:23:28.840Z30/30 Read block 43 len 24576
22572024-02-03T17:23:28.840Z----------------------------------------------------------------
22582024-02-03T17:23:28.840Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22592024-02-03T17:23:28.840ZStates: Active Active Active
22602024-02-03T17:23:28.840ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22612024-02-03T17:23:28.840Z 1 Acked 1000 Write 7 Done Done Done false
22622024-02-03T17:23:28.840Z 2 Acked 1001 Write 10 Done Done Done false
22632024-02-03T17:23:28.840Z 3 Acked 1002 Read 1 Done Done Done false
22642024-02-03T17:23:28.840Z 4 Acked 1003 Write 9 Done Done Done false
22652024-02-03T17:23:28.840Z 5 Acked 1004 Read 6 Done Done Done false
22662024-02-03T17:23:28.840Z 6 Acked 1005 Read 5 Done Done Done false
22672024-02-03T17:23:28.840Z 7 Acked 1006 Read 1 Done Done Done false
22682024-02-03T17:23:28.841Z 8 Acked 1007 Read 7 Done Done Sent false
22692024-02-03T17:23:28.841Z 9 Acked 1008 Read 8 Done Done Sent false
22702024-02-03T17:23:28.841Z 10 Acked 1009 Read 6 Done Done Sent false
22712024-02-03T17:23:28.841Z 11 Acked 1010 Flush 0 Done Done Sent false
22722024-02-03T17:23:28.841Z 12 Acked 1011 Write 5 Done Done Sent false
22732024-02-03T17:23:28.841Z 13 Acked 1012 Read 10 Done Done Sent false
22742024-02-03T17:23:28.841Z 14 Acked 1013 Write 2 Done Done Sent false
22752024-02-03T17:23:28.841Z 15 Acked 1014 Write 4 Done Done Sent false
22762024-02-03T17:23:28.841Z 16 Acked 1015 Read 10 Done Done Sent false
22772024-02-03T17:23:28.841Z 17 Acked 1016 Write 2 Done Done Sent false
22782024-02-03T17:23:28.841Z 18 Acked 1017 Read 2 Done Done Sent false
22792024-02-03T17:23:28.841Z 19 Acked 1018 Read 1 Done Done Sent false
22802024-02-03T17:23:28.842Z 20 Acked 1019 Read 7 Done Done Sent false
22812024-02-03T17:23:28.842Z 21 Acked 1020 Read 6 Done Done Sent false
22822024-02-03T17:23:28.842Z 22 Acked 1021 Read 2 Done Done Sent false
22832024-02-03T17:23:28.842Z 23 Acked 1022 Read 7 Done Done Sent false
22842024-02-03T17:23:28.842Z 24 Acked 1023 Write 2 Done Done Sent false
22852024-02-03T17:23:28.842Z 25 Acked 1024 Write 3 Done Done Sent false
22862024-02-03T17:23:28.842Z 26 Acked 1025 Write 7 Done Done Sent false
22872024-02-03T17:23:28.842Z 27 Acked 1026 Write 9 Done Done Sent false
22882024-02-03T17:23:28.842Z 28 Acked 1027 Write 10 Done Done Sent false
22892024-02-03T17:23:28.842Z 29 Acked 1028 Write 8 Done Done Sent false
22902024-02-03T17:23:28.842Z 30 Acked 1029 Read 6 Done Done Sent false
22912024-02-03T17:23:28.842Z STATES DS:0 DS:1 DS:2 TOTAL
22922024-02-03T17:23:28.842Z New 0 0 0 0
22932024-02-03T17:23:28.842Z Sent 0 0 23 23
22942024-02-03T17:23:28.843Z Done 30 30 7 67
22952024-02-03T17:23:28.843Z Skipped 0 0 0 0
22962024-02-03T17:23:28.843Z Error 0 0 0 0
22972024-02-03T17:23:28.843ZLast Flush: 1010 1010 0
22982024-02-03T17:23:28.843ZDownstairs last five completed:
22992024-02-03T17:23:28.843ZUpstairs last five completed: 30 29 28 27 26
23002024-02-03T17:23:28.843Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:27.964266298Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1209}
23012024-02-03T17:23:28.843ZWrote out file "/tmp/test_repair_verify.data"
23022024-02-03T17:23:28.843Z
23032024-02-03T17:23:28.843ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23042024-02-03T17:23:28.852ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23052024-02-03T17:23:28.852Z 0 000-019 18 18 18 30 30 30 F F F
23062024-02-03T17:23:28.852Z 1 020-039 20 20 12 32 32 21 T T T <---
23072024-02-03T17:23:28.852Z 2 040-059 20 20 12 32 32 21 F F T <---
23082024-02-03T17:23:28.852Z 3 060-079 10 10 10 17 17 17 F F F
23092024-02-03T17:23:28.852Z 4 080-099 17 17 17 28 28 28 F F F
23102024-02-03T17:23:28.852Z 5 100-119 18 18 18 29 29 29 F F F
23112024-02-03T17:23:28.853Z 6 120-139 18 18 18 30 30 30 F F F
23122024-02-03T17:23:28.853Z 7 140-159 16 16 16 27 27 27 F F F
23132024-02-03T17:23:28.857Z 8 160-179 13 13 13 22 22 22 T T F <---
23142024-02-03T17:23:28.857Z 9 180-199 19 19 19 31 31 31 F F F
23152024-02-03T17:23:28.857Z 10 200-219 16 16 16 26 26 26 T T F <---
23162024-02-03T17:23:28.857Z 11 220-239 15 15 15 24 24 24 F F F
23172024-02-03T17:23:28.857Z 12 240-259 16 16 16 27 27 27 F F F
23182024-02-03T17:23:28.857Z 13 260-279 15 15 15 24 24 24 F F F
23192024-02-03T17:23:28.857Z 14 280-299 19 19 19 31 31 31 F F F
23202024-02-03T17:23:28.857Z 15 300-319 14 14 14 23 23 23 F F F
23212024-02-03T17:23:28.857Z 16 320-339 20 20 18 32 32 30 F F T <---
23222024-02-03T17:23:28.857Z 17 340-359 19 19 19 31 31 31 F F F
23232024-02-03T17:23:28.857Z 18 360-379 19 19 19 31 31 31 F F F
23242024-02-03T17:23:28.857Z 19 380-399 19 19 19 31 31 31 F F F
23252024-02-03T17:23:28.858Z 20 400-419 18 18 18 30 30 30 T T F <---
23262024-02-03T17:23:28.858Z 21 420-439 16 16 16 27 27 27 F F F
23272024-02-03T17:23:28.858Z 22 440-459 16 16 16 27 27 27 T T F <---
23282024-02-03T17:23:28.858Z 23 460-479 19 19 19 31 31 31 F F F
23292024-02-03T17:23:28.858Z 24 480-499 13 13 13 22 22 22 F F F
23302024-02-03T17:23:28.858Z 25 500-519 16 16 16 26 26 26 F F F
23312024-02-03T17:23:28.858Z 26 520-539 10 10 10 17 17 17 T T F <---
23322024-02-03T17:23:28.858Z 27 540-559 20 20 16 32 32 25 T T T <---
23332024-02-03T17:23:28.858Z 28 560-579 16 16 16 25 25 25 T T F <---
23342024-02-03T17:23:28.858Z 29 580-599 16 16 16 27 27 27 F F F
23352024-02-03T17:23:28.858ZMax gen: 20, Max flush: 32
23362024-02-03T17:23:28.858ZError: Difference in extent metadata found!
23372024-02-03T17:23:28.868ZA Difference in extent metadata is expected here
23382024-02-03T17:23:28.869ZOn loop 10
23392024-02-03T17:23:28.869Z
23402024-02-03T17:23:28.869ZVerifying data now
23412024-02-03T17:23:34.794ZNo change after verify
23422024-02-03T17:23:34.794ZLoop: 10 Downstairs dump after verify (and repair):
23432024-02-03T17:23:34.818ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23442024-02-03T17:23:34.818Z 0 000-019 18 18 18 30 30 30 F F F
23452024-02-03T17:23:34.821Z 1 020-039 21 21 21 33 33 33 F F F
23462024-02-03T17:23:34.822Z 2 040-059 20 20 20 32 32 32 F F F
23472024-02-03T17:23:34.822Z 3 060-079 10 10 10 17 17 17 F F F
23482024-02-03T17:23:34.822Z 4 080-099 17 17 17 28 28 28 F F F
23492024-02-03T17:23:34.822Z 5 100-119 18 18 18 29 29 29 F F F
23502024-02-03T17:23:34.822Z 6 120-139 18 18 18 30 30 30 F F F
23512024-02-03T17:23:34.822Z 7 140-159 16 16 16 27 27 27 F F F
23522024-02-03T17:23:34.822Z 8 160-179 21 21 21 33 33 33 F F F
23532024-02-03T17:23:34.822Z 9 180-199 19 19 19 31 31 31 F F F
23542024-02-03T17:23:34.822Z 10 200-219 21 21 21 33 33 33 F F F
23552024-02-03T17:23:34.822Z 11 220-239 15 15 15 24 24 24 F F F
23562024-02-03T17:23:34.822Z 12 240-259 16 16 16 27 27 27 F F F
23572024-02-03T17:23:34.822Z 13 260-279 15 15 15 24 24 24 F F F
23582024-02-03T17:23:34.822Z 14 280-299 19 19 19 31 31 31 F F F
23592024-02-03T17:23:34.822Z 15 300-319 14 14 14 23 23 23 F F F
23602024-02-03T17:23:34.822Z 16 320-339 20 20 20 32 32 32 F F F
23612024-02-03T17:23:34.822Z 17 340-359 19 19 19 31 31 31 F F F
23622024-02-03T17:23:34.822Z 18 360-379 19 19 19 31 31 31 F F F
23632024-02-03T17:23:34.822Z 19 380-399 19 19 19 31 31 31 F F F
23642024-02-03T17:23:34.822Z 20 400-419 21 21 21 33 33 33 F F F
23652024-02-03T17:23:34.822Z 21 420-439 16 16 16 27 27 27 F F F
23662024-02-03T17:23:34.822Z 22 440-459 21 21 21 33 33 33 F F F
23672024-02-03T17:23:34.822Z 23 460-479 19 19 19 31 31 31 F F F
23682024-02-03T17:23:34.822Z 24 480-499 13 13 13 22 22 22 F F F
23692024-02-03T17:23:34.823Z 25 500-519 16 16 16 26 26 26 F F F
23702024-02-03T17:23:34.823Z 26 520-539 21 21 21 33 33 33 F F F
23712024-02-03T17:23:34.823Z 27 540-559 21 21 21 33 33 33 F F F
23722024-02-03T17:23:34.823Z 28 560-579 21 21 21 33 33 33 F F F
23732024-02-03T17:23:34.823Z 29 580-599 16 16 16 27 27 27 F F F
23742024-02-03T17:23:34.823ZMax gen: 21, Max flush: 33
23752024-02-03T17:23:34.826Z
23762024-02-03T17:23:34.826ZBegin loop 11
23772024-02-03T17:23:34.826ZDownstairs to restart: 1
23782024-02-03T17:23:34.864Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.013861201Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217}
23792024-02-03T17:23:34.864Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.014460321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217}
23802024-02-03T17:23:34.864Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.014510156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217}
23812024-02-03T17:23:34.875Z{"msg":"Crucible aa37671f-343e-4f0a-9b81-33b8818e95ef has session id: 71e13263-ba78-4e07-a42a-db47c0a7840c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.01455007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23822024-02-03T17:23:34.875Z{"msg":"Upstairs opts: Upstairs UUID: aa37671f-343e-4f0a-9b81-33b8818e95ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.01460712Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23832024-02-03T17:23:34.876Z{"msg":"Crucible stats registered with UUID: aa37671f-343e-4f0a-9b81-33b8818e95ef","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.014652598Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23842024-02-03T17:23:34.876ZCrucible runtime is spawned
23852024-02-03T17:23:34.876Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.014966987Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217}
23862024-02-03T17:23:34.876Z{"msg":"aa37671f-343e-4f0a-9b81-33b8818e95ef active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015113209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23872024-02-03T17:23:34.876Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015227563Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23882024-02-03T17:23:34.876Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015274289Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23892024-02-03T17:23:34.876Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015316295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23902024-02-03T17:23:34.876Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015391323Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23912024-02-03T17:23:34.876Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015523688Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23922024-02-03T17:23:34.876Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015666341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23932024-02-03T17:23:34.876Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015716766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23942024-02-03T17:23:34.876Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015762314Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23952024-02-03T17:23:34.876Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015858836Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23962024-02-03T17:23:34.876Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.015909329Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23972024-02-03T17:23:34.876Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.016113936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23982024-02-03T17:23:34.876Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.016145316Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
23992024-02-03T17:23:34.876Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.017142039Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24002024-02-03T17:23:34.876Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.017225122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24012024-02-03T17:23:34.876Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.017270494Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24022024-02-03T17:23:34.876Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.017313984Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24032024-02-03T17:23:34.876Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.018298984Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24042024-02-03T17:23:34.876Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.018429679Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24052024-02-03T17:23:34.876Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.018495067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24062024-02-03T17:23:34.877Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.018523967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24072024-02-03T17:23:34.877Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.019274678Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24082024-02-03T17:23:34.877Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.019368303Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24092024-02-03T17:23:34.877Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.019450526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24102024-02-03T17:23:34.877Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.019529869Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24112024-02-03T17:23:34.877Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.019606775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24122024-02-03T17:23:34.877Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.019677854Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24132024-02-03T17:23:34.877Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.019901298Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24142024-02-03T17:23:34.877Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020580748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24152024-02-03T17:23:34.877Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020619355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24162024-02-03T17:23:34.877Z{"msg":"[0]R flush_numbers[0..12]: [30, 33, 32, 17, 28, 29, 30, 27, 33, 31, 33, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020644083Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24172024-02-03T17:23:34.877Z{"msg":"[0]R generation[0..12]: [18, 21, 20, 10, 17, 18, 18, 16, 21, 19, 21, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020667099Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24182024-02-03T17:23:34.877Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020756262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24192024-02-03T17:23:34.877Z{"msg":"[1]R flush_numbers[0..12]: [30, 33, 32, 17, 28, 29, 30, 27, 33, 31, 33, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020808538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24202024-02-03T17:23:34.877Z{"msg":"[1]R generation[0..12]: [18, 21, 20, 10, 17, 18, 18, 16, 21, 19, 21, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020835099Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24212024-02-03T17:23:34.877Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020878453Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24222024-02-03T17:23:34.877Z{"msg":"[2]R flush_numbers[0..12]: [30, 33, 32, 17, 28, 29, 30, 27, 33, 31, 33, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.020903907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24232024-02-03T17:23:34.877Z{"msg":"[2]R generation[0..12]: [18, 21, 20, 10, 17, 18, 18, 16, 21, 19, 21, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.02095003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24242024-02-03T17:23:34.877Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO
24252024-02-03T17:23:34.877Z"level":30,"time":"2024-02-03T17:23:34.020981465Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24262024-02-03T17:23:34.877Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.02102324Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24272024-02-03T17:23:34.877Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021063186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24282024-02-03T17:23:34.877Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.02110863Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24292024-02-03T17:23:34.877Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021133089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24302024-02-03T17:23:34.878Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.0211749Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24312024-02-03T17:23:34.878Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021197806Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24322024-02-03T17:23:34.878Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021219645Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24332024-02-03T17:23:34.878Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021260728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24342024-02-03T17:23:34.878Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021284795Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24352024-02-03T17:23:34.878Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021324641Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24362024-02-03T17:23:34.878Z{"msg":"aa37671f-343e-4f0a-9b81-33b8818e95ef is now active with session: 71e13263-ba78-4e07-a42a-db47c0a7840c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.02134792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24372024-02-03T17:23:34.878Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021386951Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217,"session_id":"71e13263-ba78-4e07-a42a-db47c0a7840c"}
24382024-02-03T17:23:34.878Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:34.021409228Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217}
24392024-02-03T17:23:34.878ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24402024-02-03T17:23:34.878ZLoading write count information from file "/tmp/test_repair_verify.data"
24412024-02-03T17:23:34.878ZRun Repair workload
24422024-02-03T17:23:34.878Z01/30 Write block 82 len 16384 data: 2 3 3 3
24432024-02-03T17:23:34.878Z02/30 Write block 404 len 40960 data: 3 3 3 3 3 3 3 3 3 3
24442024-02-03T17:23:34.878Z03/30 Read block 415 len 8192
24452024-02-03T17:23:34.882Z04/30 Write block 535 len 32768 data: 2 2 2 2 2 3 3 3
24462024-02-03T17:23:34.882Z05/30 Flush
24472024-02-03T17:23:34.882Z06/30 Write block 377 len 12288 data: 4 4 3
24482024-02-03T17:23:34.882Z07/30 Read block 461 len 40960
24492024-02-03T17:23:34.882Z08/30 Read block 522 len 20480
24502024-02-03T17:23:34.882Z09/30 Write block 153 len 20480 data: 4 5 4 4 4
24512024-02-03T17:23:34.888Z10/30 Write block 544 len 16384 data: 2 2 2 2
24522024-02-03T17:23:34.888Z11/30 Read block 300 len 40960
24532024-02-03T17:23:34.892Z12/30 Write block 287 len 32768 data: 3 3 3 3 3 4 4 3
24542024-02-03T17:23:34.892Z13/30 Write block 39 len 12288 data: 4 4 3
24552024-02-03T17:23:34.892Z14/30 Write block 576 len 36864 data: 3 4 4 5 4 4 4 4 3
24562024-02-03T17:23:34.892Z15/30 Read block 154 len 40960
24572024-02-03T17:23:34.895Z16/30 Read block 580 len 16384
24582024-02-03T17:23:34.895Z17/30 Read block 436 len 40960
24592024-02-03T17:23:34.895Z18/30 Read block 538 len 20480
24602024-02-03T17:23:34.898Z19/30 Read block 266 len 8192
24612024-02-03T17:23:34.898Z20/30 Write block 571 len 20480 data: 4 3 2 3 3
24622024-02-03T17:23:34.902Z21/30 Write block 328 len 32768 data: 4 3 4 4 4 5 4 3
24632024-02-03T17:23:34.902Z22/30 Read block 34 len 12288
24642024-02-03T17:23:34.902Z23/30 Read block 195 len 20480
24652024-02-03T17:23:34.905Z24/30 Read block 263 len 40960
24662024-02-03T17:23:34.905Z25/30 Read block 316 len 4096
24672024-02-03T17:23:34.908Z26/30 Write block 255 len 40960 data: 4 4 2 2 2 3 3 3 3 3
24682024-02-03T17:23:34.908Z27/30 Write block 266 len 12288 data: 3 3 4
24692024-02-03T17:23:34.912Z28/30 Write block 350 len 20480 data: 3 3 3 2 2
24702024-02-03T17:23:34.912Z29/30 Write block 582 len 4096 data: 5
24712024-02-03T17:23:34.912Z30/30 Read block 544 len 4096
24722024-02-03T17:23:34.912Z----------------------------------------------------------------
24732024-02-03T17:23:34.912Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24742024-02-03T17:23:34.912ZStates: Active Active Active
24752024-02-03T17:23:34.912ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24762024-02-03T17:23:34.912Z 1 Acked 1000 Write 4 Done Sent Done false
24772024-02-03T17:23:34.912Z 2 Acked 1001 Write 10 Done Sent Done false
24782024-02-03T17:23:34.912Z 3 Acked 1002 Read 2 Done Sent Done false
24792024-02-03T17:23:34.912Z 4 Acked 1003 Write 8 Done Sent Done false
24802024-02-03T17:23:34.912Z 5 Acked 1004 Flush 0 Done Sent Done false
24812024-02-03T17:23:34.912Z 6 Acked 1005 Write 3 Done Sent Done false
24822024-02-03T17:23:34.912Z 7 Acked 1006 Read 10 Done Sent Done false
24832024-02-03T17:23:34.912Z 8 Acked 1007 Read 5 Done Sent Done false
24842024-02-03T17:23:34.912Z 9 Acked 1008 Write 5 Done Sent Done false
24852024-02-03T17:23:34.912Z 10 Acked 1009 Write 4 Done Sent Done false
24862024-02-03T17:23:34.912Z 11 Acked 1010 Read 10 Done Sent Done false
24872024-02-03T17:23:34.912Z 12 Acked 1011 Write 8 Done Sent Done false
24882024-02-03T17:23:34.912Z 13 Acked 1012 Write 3 Done Sent Done false
24892024-02-03T17:23:34.912Z 14 Acked 1013 Write 9 Done Sent Done false
24902024-02-03T17:23:34.912Z 15 Acked 1014 Read 10 Done Sent Done false
24912024-02-03T17:23:34.912Z 16 Acked 1015 Read 4 Done Sent Done false
24922024-02-03T17:23:34.912Z 17 Acked 1016 Read 10 Done Sent Done false
24932024-02-03T17:23:34.913Z 18 Acked 1017 Read 5 Done Sent Done false
24942024-02-03T17:23:34.913Z 19 Acked 1018 Read 2 Done Sent Done false
24952024-02-03T17:23:34.913Z 20 Acked 1019 Write 5 Done Sent Done false
24962024-02-03T17:23:34.913Z 21 Acked 1020 Write 8 Done Sent Done false
24972024-02-03T17:23:34.913Z 22 Acked 1021 Read 3 Done Sent Done false
24982024-02-03T17:23:34.913Z 23 Acked 1022 Read 5 Done Sent Done false
24992024-02-03T17:23:34.913Z 24 Acked 1023 Read 10 Done Sent Done false
25002024-02-03T17:23:34.913Z 25 Acked 1024 Read 1 Done Sent Done false
25012024-02-03T17:23:34.913Z 26 Acked 1025 Write 10 Done Sent Done false
25022024-02-03T17:23:34.913Z 27 Acked 1026 Write 3 Done Sent Done false
25032024-02-03T17:23:34.913Z 28 Acked 1027 Write 5 Done Sent Done false
25042024-02-03T17:23:34.913Z 29 Acked 1028 Write 1 Done Sent Done false
25052024-02-03T17:23:34.913Z 30 Acked 1029 Read 1 Done Sent Sent false
25062024-02-03T17:23:34.913Z STATES DS:0 DS:1 DS:2 TOTAL
25072024-02-03T17:23:34.913Z New 0 0 0 0
25082024-02-03T17:23:34.913Z Sent 0 30 1 31
25092024-02-03T17:23:34.913Z Done 30 0 29 59
25102024-02-03T17:23:34.913Z Skipped 0 0 0 0
25112024-02-03T17:23:34.913Z Error 0 0 0 0
25122024-02-03T17:23:34.913ZLast Flush: 1004 0 1004
25132024-02-03T17:23:34.913ZDownstairs last five completed:
25142024-02-03T17:23:34.913ZUpstairs last five completed: 30 29 28 27 26
25152024-02-03T17:23:34.917Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:34.069570533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1217}
25162024-02-03T17:23:34.917ZWrote out file "/tmp/test_repair_verify.data"
25172024-02-03T17:23:34.920Z
25182024-02-03T17:23:34.927ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25192024-02-03T17:23:34.958ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25202024-02-03T17:23:34.958Z 0 000-019 18 18 18 30 30 30 F F F
25212024-02-03T17:23:34.958Z 1 020-039 21 21 21 33 33 33 T F T <---
25222024-02-03T17:23:34.958Z 2 040-059 20 20 20 32 32 32 T F T <---
25232024-02-03T17:23:34.958Z 3 060-079 10 10 10 17 17 17 F F F
25242024-02-03T17:23:34.962Z 4 080-099 22 17 22 34 28 34 F F F <---
25252024-02-03T17:23:34.962Z 5 100-119 18 18 18 29 29 29 F F F
25262024-02-03T17:23:34.962Z 6 120-139 18 18 18 30 30 30 F F F
25272024-02-03T17:23:34.962Z 7 140-159 16 16 16 27 27 27 T F T <---
25282024-02-03T17:23:34.962Z 8 160-179 21 21 21 33 33 33 F F F
25292024-02-03T17:23:34.962Z 9 180-199 19 19 19 31 31 31 F F F
25302024-02-03T17:23:34.962Z 10 200-219 21 21 21 33 33 33 F F F
25312024-02-03T17:23:34.962Z 11 220-239 15 15 15 24 24 24 F F F
25322024-02-03T17:23:34.962Z 12 240-259 16 16 16 27 27 27 T F T <---
25332024-02-03T17:23:34.962Z 13 260-279 15 15 15 24 24 24 T F T <---
25342024-02-03T17:23:34.962Z 14 280-299 19 19 19 31 31 31 T F T <---
25352024-02-03T17:23:34.962Z 15 300-319 14 14 14 23 23 23 F F F
25362024-02-03T17:23:34.962Z 16 320-339 20 20 20 32 32 32 T F T <---
25372024-02-03T17:23:34.962Z 17 340-359 19 19 19 31 31 31 T F T <---
25382024-02-03T17:23:34.962Z 18 360-379 19 19 19 31 31 31 T F T <---
25392024-02-03T17:23:34.962Z 19 380-399 19 19 19 31 31 31 F F F
25402024-02-03T17:23:34.962Z 20 400-419 22 21 22 34 33 34 F F F <---
25412024-02-03T17:23:34.962Z 21 420-439 16 16 16 27 27 27 F F F
25422024-02-03T17:23:34.962Z 22 440-459 21 21 21 33 33 33 F F F
25432024-02-03T17:23:34.963Z 23 460-479 19 19 19 31 31 31 F F F
25442024-02-03T17:23:34.963Z 24 480-499 13 13 13 22 22 22 F F F
25452024-02-03T17:23:34.963Z 25 500-519 16 16 16 26 26 26 F F F
25462024-02-03T17:23:34.963Z 26 520-539 22 21 22 34 33 34 F F F <---
25472024-02-03T17:23:34.963Z 27 540-559 22 21 22 34 33 34 T F T <---
25482024-02-03T17:23:34.963Z 28 560-579 21 21 21 33 33 33 T F T <---
25492024-02-03T17:23:34.963Z 29 580-599 16 16 16 27 27 27 T F T <---
25502024-02-03T17:23:34.963ZMax gen: 22, Max flush: 34
25512024-02-03T17:23:34.963ZError: Difference in extent metadata found!
25522024-02-03T17:23:34.966ZA Difference in extent metadata is expected here
25532024-02-03T17:23:34.966ZOn loop 11
25542024-02-03T17:23:34.966Z
25552024-02-03T17:23:34.969ZVerifying data now
25562024-02-03T17:23:40.297ZNo change after verify
25572024-02-03T17:23:40.297ZLoop: 11 Downstairs dump after verify (and repair):
25582024-02-03T17:23:40.322ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25592024-02-03T17:23:40.323Z 0 000-019 18 18 18 30 30 30 F F F
25602024-02-03T17:23:40.323Z 1 020-039 23 23 23 35 35 35 F F F
25612024-02-03T17:23:40.323Z 2 040-059 23 23 23 35 35 35 F F F
25622024-02-03T17:23:40.327Z 3 060-079 10 10 10 17 17 17 F F F
25632024-02-03T17:23:40.327Z 4 080-099 22 22 22 34 34 34 F F F
25642024-02-03T17:23:40.327Z 5 100-119 18 18 18 29 29 29 F F F
25652024-02-03T17:23:40.327Z 6 120-139 18 18 18 30 30 30 F F F
25662024-02-03T17:23:40.327Z 7 140-159 23 23 23 35 35 35 F F F
25672024-02-03T17:23:40.328Z 8 160-179 21 21 21 33 33 33 F F F
25682024-02-03T17:23:40.328Z 9 180-199 19 19 19 31 31 31 F F F
25692024-02-03T17:23:40.328Z 10 200-219 21 21 21 33 33 33 F F F
25702024-02-03T17:23:40.328Z 11 220-239 15 15 15 24 24 24 F F F
25712024-02-03T17:23:40.328Z 12 240-259 23 23 23 35 35 35 F F F
25722024-02-03T17:23:40.328Z 13 260-279 23 23 23 35 35 35 F F F
25732024-02-03T17:23:40.328Z 14 280-299 23 23 23 35 35 35 F F F
25742024-02-03T17:23:40.328Z 15 300-319 14 14 14 23 23 23 F F F
25752024-02-03T17:23:40.328Z 16 320-339 23 23 23 35 35 35 F F F
25762024-02-03T17:23:40.328Z 17 340-359 23 23 23 35 35 35 F F F
25772024-02-03T17:23:40.328Z 18 360-379 23 23 23 35 35 35 F F F
25782024-02-03T17:23:40.328Z 19 380-399 19 19 19 31 31 31 F F F
25792024-02-03T17:23:40.328Z 20 400-419 22 22 22 34 34 34 F F F
25802024-02-03T17:23:40.329Z 21 420-439 16 16 16 27 27 27 F F F
25812024-02-03T17:23:40.329Z 22 440-459 21 21 21 33 33 33 F F F
25822024-02-03T17:23:40.329Z 23 460-479 19 19 19 31 31 31 F F F
25832024-02-03T17:23:40.329Z 24 480-499 13 13 13 22 22 22 F F F
25842024-02-03T17:23:40.329Z 25 500-519 16 16 16 26 26 26 F F F
25852024-02-03T17:23:40.329Z 26 520-539 22 22 22 34 34 34 F F F
25862024-02-03T17:23:40.329Z 27 540-559 23 23 23 35 35 35 F F F
25872024-02-03T17:23:40.329Z 28 560-579 23 23 23 35 35 35 F F F
25882024-02-03T17:23:40.329Z 29 580-599 23 23 23 35 35 35 F F F
25892024-02-03T17:23:40.329ZMax gen: 23, Max flush: 35
25902024-02-03T17:23:40.329Z
25912024-02-03T17:23:40.329ZBegin loop 12
25922024-02-03T17:23:40.329ZDownstairs to restart: 2
25932024-02-03T17:23:40.361Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.517872349Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225}
25942024-02-03T17:23:40.367Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.518449502Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225}
25952024-02-03T17:23:40.367Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.518587339Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225}
25962024-02-03T17:23:40.367Z{"msg":"Crucible d5ab9aca-7712-4d63-a5f3-1e07fec76354 has session id: 70ab31c5-681b-4dab-9e61-a16b9f795d02","v":0,"name":"crucible","level":30Crucible runtime is spawned
25972024-02-03T17:23:40.367Z,"time":"2024-02-03T17:23:39.518655265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
25982024-02-03T17:23:40.367Z{"msg":"Upstairs opts: Upstairs UUID: d5ab9aca-7712-4d63-a5f3-1e07fec76354, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.518757023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
25992024-02-03T17:23:40.368Z{"msg":"Crucible stats registered with UUID: d5ab9aca-7712-4d63-a5f3-1e07fec76354","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.518800255Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26002024-02-03T17:23:40.368Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.518860041Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225}
26012024-02-03T17:23:40.368Z{"msg":"d5ab9aca-7712-4d63-a5f3-1e07fec76354 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519000961Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26022024-02-03T17:23:40.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519056272Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26032024-02-03T17:23:40.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519083376Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26042024-02-03T17:23:40.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519133941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26052024-02-03T17:23:40.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519159567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26062024-02-03T17:23:40.368Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519182072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26072024-02-03T17:23:40.368Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519219915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26082024-02-03T17:23:40.368Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519288553Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26092024-02-03T17:23:40.368Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519336967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26102024-02-03T17:23:40.368Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519415597Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26112024-02-03T17:23:40.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519525939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26122024-02-03T17:23:40.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519602234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26132024-02-03T17:23:40.369Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.519741591Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26142024-02-03T17:23:40.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.52090216Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26152024-02-03T17:23:40.369Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.520978457Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26162024-02-03T17:23:40.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.521112686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26172024-02-03T17:23:40.369Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.521207526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26182024-02-03T17:23:40.369Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.52202609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26192024-02-03T17:23:40.369Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.522285097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26202024-02-03T17:23:40.369Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.522368508Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26212024-02-03T17:23:40.369Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.522408875Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26222024-02-03T17:23:40.369Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.522882997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26232024-02-03T17:23:40.370Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.522951183Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26242024-02-03T17:23:40.370Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.522995848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26252024-02-03T17:23:40.370Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.523231571Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26262024-02-03T17:23:40.370Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.523290628Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26272024-02-03T17:23:40.370Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.523334387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26282024-02-03T17:23:40.370Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.523395538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26292024-02-03T17:23:40.375Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524080586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26302024-02-03T17:23:40.375Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.52416051Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26312024-02-03T17:23:40.375Z{"msg":"[0]R flush_numbers[0..12]: [30, 35, 35, 17, 34, 29, 30, 35, 33, 31, 33, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.52422739Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26322024-02-03T17:23:40.375Z{"msg":"[0]R generation[0..12]: [18, 23, 23, 10, 22, 18, 18, 23, 21, 19, 21, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524296484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26332024-02-03T17:23:40.375Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524368292Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26342024-02-03T17:23:40.375Z{"msg":"Wait for a query_work_queue command to finish before sending IO
26352024-02-03T17:23:40.375Z[1]R flush_numbers[0..12]: [30, 35, 35, 17, 34, 29, 30, 35, 33, 31, 33, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524413291Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26362024-02-03T17:23:40.375Z{"msg":"[1]R generation[0..12]: [18, 23, 23, 10, 22, 18, 18, 23, 21, 19, 21, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524452822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26372024-02-03T17:23:40.375Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524498314Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26382024-02-03T17:23:40.375Z{"msg":"[2]R flush_numbers[0..12]: [30, 35, 35, 17, 34, 29, 30, 35, 33, 31, 33, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524534294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26392024-02-03T17:23:40.375Z{"msg":"[2]R generation[0..12]: [18, 23, 23, 10, 22, 18, 18, 23, 21, 19, 21, 15]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524567185Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26402024-02-03T17:23:40.376Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.52468241Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26412024-02-03T17:23:40.376Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524707693Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26422024-02-03T17:23:40.376Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524730087Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26432024-02-03T17:23:40.376Z{"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524751403Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26442024-02-03T17:23:40.376Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524773072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26452024-02-03T17:23:40.376Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.5247949Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26462024-02-03T17:23:40.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524815139Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26472024-02-03T17:23:40.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524838311Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26482024-02-03T17:23:40.376Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.52488673Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26492024-02-03T17:23:40.376Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524910737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26502024-02-03T17:23:40.376Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.52493149Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26512024-02-03T17:23:40.376Z{"msg":"d5ab9aca-7712-4d63-a5f3-1e07fec76354 is now active with session: 70ab31c5-681b-4dab-9e61-a16b9f795d02","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524972246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26522024-02-03T17:23:40.376Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.524994746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225}
26532024-02-03T17:23:40.377Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:39.525012313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225,"session_id":"70ab31c5-681b-4dab-9e61-a16b9f795d02"}
26542024-02-03T17:23:40.377ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26552024-02-03T17:23:40.377ZLoading write count information from file "/tmp/test_repair_verify.data"
26562024-02-03T17:23:40.377ZRun Repair workload
26572024-02-03T17:23:40.377Z01/30 Write block 187 len 20480 data: 3 3 3 2 2
26582024-02-03T17:23:40.377Z02/30 Write block 87 len 4096 data: 3
26592024-02-03T17:23:40.377Z03/30 Write block 49 len 16384 data: 4 4 4 4
26602024-02-03T17:23:40.377Z04/30 Write block 95 len 24576 data: 3 3 2 2 2 2
26612024-02-03T17:23:40.377Z05/30 Write block 509 len 8192 data: 5 3
26622024-02-03T17:23:40.377Z06/30 Flush
26632024-02-03T17:23:40.382Z07/30 Write block 469 len 24576 data: 4 3 3 3 2 3
26642024-02-03T17:23:40.382Z08/30 Write block 445 len 16384 data: 4 4 4 5
26652024-02-03T17:23:40.382Z09/30 Read block 217 len 36864
26662024-02-03T17:23:40.383Z10/30 Read block 146 len 24576
26672024-02-03T17:23:40.390Z11/30 Flush
26682024-02-03T17:23:40.390Z12/30 Flush
26692024-02-03T17:23:40.393Z13/30 Write block 200 len 16384 data: 3 2 2 2
26702024-02-03T17:23:40.393Z14/30 Read block 43 len 24576
26712024-02-03T17:23:40.393Z15/30 Write block 20 len 24576 data: 2 3 3 3 3 3
26722024-02-03T17:23:40.393Z16/30 Flush
26732024-02-03T17:23:40.397Z17/30 Write block 260 len 12288 data: 4 4 4
26742024-02-03T17:23:40.397Z18/30 Read block 134 len 28672
26752024-02-03T17:23:40.397Z19/30 Read block 239 len 28672
26762024-02-03T17:23:40.400Z20/30 Write block 107 len 36864 data: 3 3 4 4 4 4 4 5 4
26772024-02-03T17:23:40.401Z21/30 Write block 347 len 28672 data: 5 3 3 4 4 4 3
26782024-02-03T17:23:40.409Z22/30 Read block 303 len 32768
26792024-02-03T17:23:40.409Z23/30 Write block 566 len 20480 data: 4 4 4 3 4
26802024-02-03T17:23:40.409Z24/30 Flush
26812024-02-03T17:23:40.413Z25/30 Write block 528 len 8192 data: 5 6
26822024-02-03T17:23:40.413Z26/30 Write block 227 len 4096 data: 4
26832024-02-03T17:23:40.413Z27/30 Write block 235 len 12288 data: 3 3 3
26842024-02-03T17:23:40.413Z28/30 Read block 158 len 36864
26852024-02-03T17:23:40.413Z29/30 Read block 56 len 8192
26862024-02-03T17:23:40.417Z30/30 Write block 278 len 28672 data: 4 2 2 3 3 3 3
26872024-02-03T17:23:40.417Z----------------------------------------------------------------
26882024-02-03T17:23:40.417Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
26892024-02-03T17:23:40.417ZStates: Active Active Active
26902024-02-03T17:23:40.417ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26912024-02-03T17:23:40.417Z 1 Acked 1000 Write 5 Done Done Sent false
26922024-02-03T17:23:40.417Z 2 Acked 1001 Write 1 Done Done Sent false
26932024-02-03T17:23:40.417Z 3 Acked 1002 Write 4 Done Done Sent false
26942024-02-03T17:23:40.417Z 4 Acked 1003 Write 6 Done Done Sent false
26952024-02-03T17:23:40.418Z 5 Acked 1004 Write 2 Done Done Sent false
26962024-02-03T17:23:40.418Z 6 Acked 1005 Flush 0 Done Done Sent false
26972024-02-03T17:23:40.418Z 7 Acked 1006 Write 6 Done Done Sent false
26982024-02-03T17:23:40.418Z 8 Acked 1007 Write 4 Done Done Sent false
26992024-02-03T17:23:40.418Z 9 Acked 1008 Read 9 Done Done Sent false
27002024-02-03T17:23:40.418Z 10 Acked 1009 Read 6 Done Done Sent false
27012024-02-03T17:23:40.418Z 11 Acked 1010 Flush 0 Done Done Sent false
27022024-02-03T17:23:40.418Z 12 Acked 1011 Flush 0 Done Done Sent false
27032024-02-03T17:23:40.418Z 13 Acked 1012 Write 4 Done Done Sent false
27042024-02-03T17:23:40.418Z 14 Acked 1013 Read 6 Done Done Sent false
27052024-02-03T17:23:40.418Z 15 Acked 1014 Write 6 Done Done Sent false
27062024-02-03T17:23:40.418Z 16 Acked 1015 Flush 0 Done Done Sent false
27072024-02-03T17:23:40.418Z 17 Acked 1016 Write 3 Done Done Sent false
27082024-02-03T17:23:40.419Z 18 Acked 1017 Read 7 Done Done Sent false
27092024-02-03T17:23:40.419Z 19 Acked 1018 Read 7 Done Done Sent false
27102024-02-03T17:23:40.419Z 20 Acked 1019 Write 9 Done Done Sent false
27112024-02-03T17:23:40.419Z 21 Acked 1020 Write 7 Done Done Sent false
27122024-02-03T17:23:40.419Z 22 Acked 1021 Read 8 Done Done Sent false
27132024-02-03T17:23:40.419Z 23 Acked 1022 Write 5 Done Done Sent false
27142024-02-03T17:23:40.419Z 24 Acked 1023 Flush 0 Done Done Sent false
27152024-02-03T17:23:40.419Z 25 Acked 1024 Write 2 Done Done Sent false
27162024-02-03T17:23:40.419Z 26 Acked 1025 Write 1 Done Done Sent false
27172024-02-03T17:23:40.419Z 27 Acked 1026 Write 3 Done Done Sent false
27182024-02-03T17:23:40.419Z 28 Acked 1027 Read 9 Done Done Sent false
27192024-02-03T17:23:40.419Z 29 Acked 1028 Read 2 Done Done Sent false
27202024-02-03T17:23:40.419Z 30 Acked 1029 Write 7 Sent Sent Sent false
27212024-02-03T17:23:40.420Z STATES DS:0 DS:1 DS:2 TOTAL
27222024-02-03T17:23:40.420Z New 0 0 0 0
27232024-02-03T17:23:40.420Z Sent 1 1 30 32
27242024-02-03T17:23:40.420Z Done 29 29 0 58
27252024-02-03T17:23:40.420Z Skipped 0 0 0 0
27262024-02-03T17:23:40.420Z Error 0 0 0 0
27272024-02-03T17:23:40.420ZLast Flush: 1023 1023 0
27282024-02-03T17:23:40.420ZDownstairs last five completed:
27292024-02-03T17:23:40.420ZUpstairs last five completed: 30 29 28 27 26
27302024-02-03T17:23:40.420Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:39.572688865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1225}
27312024-02-03T17:23:40.420ZWrote out file "/tmp/test_repair_verify.data"
27322024-02-03T17:23:40.424Z
27332024-02-03T17:23:40.430ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27342024-02-03T17:23:40.475ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27352024-02-03T17:23:40.475Z 0 000-019 18 18 18 30 30 30 F F F
27362024-02-03T17:23:40.479Z 1 020-039 24 24 23 39 39 35 F F F <---
27372024-02-03T17:23:40.479Z 2 040-059 24 24 23 36 36 35 F F F <---
27382024-02-03T17:23:40.479Z 3 060-079 10 10 10 17 17 17 F F F
27392024-02-03T17:23:40.479Z 4 080-099 24 24 22 36 36 34 F F F <---
27402024-02-03T17:23:40.479Z 5 100-119 24 24 18 40 40 29 F F F <---
27412024-02-03T17:23:40.479Z 6 120-139 18 18 18 30 30 30 F F F
27422024-02-03T17:23:40.480Z 7 140-159 23 23 23 35 35 35 F F F
27432024-02-03T17:23:40.480Z 8 160-179 21 21 21 33 33 33 F F F
27442024-02-03T17:23:40.480Z 9 180-199 24 24 19 36 36 31 F F F <---
27452024-02-03T17:23:40.480Z 10 200-219 24 24 21 39 39 33 F F F <---
27462024-02-03T17:23:40.480Z 11 220-239 15 15 15 24 24 24 T T F <---
27472024-02-03T17:23:40.480Z 12 240-259 23 23 23 35 35 35 F F F
27482024-02-03T17:23:40.480Z 13 260-279 24 24 23 40 40 35 T T F <---
27492024-02-03T17:23:40.480Z 14 280-299 23 23 23 35 35 35 T T F <---
27502024-02-03T17:23:40.480Z 15 300-319 14 14 14 23 23 23 F F F
27512024-02-03T17:23:40.480Z 16 320-339 23 23 23 35 35 35 F F F
27522024-02-03T17:23:40.480Z 17 340-359 24 24 23 40 40 35 F F F <---
27532024-02-03T17:23:40.480Z 18 360-379 23 23 23 35 35 35 F F F
27542024-02-03T17:23:40.480Z 19 380-399 19 19 19 31 31 31 F F F
27552024-02-03T17:23:40.481Z 20 400-419 22 22 22 34 34 34 F F F
27562024-02-03T17:23:40.481Z 21 420-439 16 16 16 27 27 27 F F F
27572024-02-03T17:23:40.481Z 22 440-459 24 24 21 37 37 33 F F F <---
27582024-02-03T17:23:40.481Z 23 460-479 24 24 19 37 37 31 F F F <---
27592024-02-03T17:23:40.481Z 24 480-499 13 13 13 22 22 22 F F F
27602024-02-03T17:23:40.481Z 25 500-519 24 24 16 36 36 26 F F F <---
27612024-02-03T17:23:40.481Z 26 520-539 22 22 22 34 34 34 T T F <---
27622024-02-03T17:23:40.481Z 27 540-559 23 23 23 35 35 35 F F F
27632024-02-03T17:23:40.481Z 28 560-579 24 24 23 40 40 35 F F F <---
27642024-02-03T17:23:40.481Z 29 580-599 23 23 23 35 35 35 F F F
27652024-02-03T17:23:40.481ZMax gen: 24, Max flush: 40
27662024-02-03T17:23:40.481ZError: Difference in extent metadata found!
27672024-02-03T17:23:40.481ZA Difference in extent metadata is expected here
27682024-02-03T17:23:40.482ZOn loop 12
27692024-02-03T17:23:40.482Z
27702024-02-03T17:23:40.482ZVerifying data now
27712024-02-03T17:23:45.740ZNo change after verify
27722024-02-03T17:23:45.740ZLoop: 12 Downstairs dump after verify (and repair):
27732024-02-03T17:23:45.764ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27742024-02-03T17:23:45.764Z 0 000-019 18 18 18 30 30 30 F F F
27752024-02-03T17:23:45.768Z 1 020-039 24 24 24 39 39 39 F F F
27762024-02-03T17:23:45.768Z 2 040-059 24 24 24 36 36 36 F F F
27772024-02-03T17:23:45.768Z 3 060-079 10 10 10 17 17 17 F F F
27782024-02-03T17:23:45.768Z 4 080-099 24 24 24 36 36 36 F F F
27792024-02-03T17:23:45.768Z 5 100-119 24 24 24 40 40 40 F F F
27802024-02-03T17:23:45.768Z 6 120-139 18 18 18 30 30 30 F F F
27812024-02-03T17:23:45.768Z 7 140-159 23 23 23 35 35 35 F F F
27822024-02-03T17:23:45.768Z 8 160-179 21 21 21 33 33 33 F F F
27832024-02-03T17:23:45.768Z 9 180-199 24 24 24 36 36 36 F F F
27842024-02-03T17:23:45.768Z 10 200-219 24 24 24 39 39 39 F F F
27852024-02-03T17:23:45.768Z 11 220-239 25 25 25 41 41 41 F F F
27862024-02-03T17:23:45.768Z 12 240-259 23 23 23 35 35 35 F F F
27872024-02-03T17:23:45.768Z 13 260-279 25 25 25 41 41 41 F F F
27882024-02-03T17:23:45.768Z 14 280-299 25 25 25 41 41 41 F F F
27892024-02-03T17:23:45.768Z 15 300-319 14 14 14 23 23 23 F F F
27902024-02-03T17:23:45.768Z 16 320-339 23 23 23 35 35 35 F F F
27912024-02-03T17:23:45.768Z 17 340-359 24 24 24 40 40 40 F F F
27922024-02-03T17:23:45.768Z 18 360-379 23 23 23 35 35 35 F F F
27932024-02-03T17:23:45.768Z 19 380-399 19 19 19 31 31 31 F F F
27942024-02-03T17:23:45.768Z 20 400-419 22 22 22 34 34 34 F F F
27952024-02-03T17:23:45.768Z 21 420-439 16 16 16 27 27 27 F F F
27962024-02-03T17:23:45.768Z 22 440-459 24 24 24 37 37 37 F F F
27972024-02-03T17:23:45.768Z 23 460-479 24 24 24 37 37 37 F F F
27982024-02-03T17:23:45.769Z 24 480-499 13 13 13 22 22 22 F F F
27992024-02-03T17:23:45.769Z 25 500-519 24 24 24 36 36 36 F F F
28002024-02-03T17:23:45.769Z 26 520-539 25 25 25 41 41 41 F F F
28012024-02-03T17:23:45.769Z 27 540-559 23 23 23 35 35 35 F F F
28022024-02-03T17:23:45.769Z 28 560-579 24 24 24 40 40 40 F F F
28032024-02-03T17:23:45.769Z 29 580-599 23 23 23 35 35 35 F F F
28042024-02-03T17:23:45.769ZMax gen: 25, Max flush: 41
28052024-02-03T17:23:45.772Z
28062024-02-03T17:23:45.772ZBegin loop 13
28072024-02-03T17:23:45.772ZDownstairs to restart: 0
28082024-02-03T17:23:45.802Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.959947675Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233}
28092024-02-03T17:23:45.802Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.960497688Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233}
28102024-02-03T17:23:45.806Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.960593302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233}
28112024-02-03T17:23:45.806Z{"msg":"Crucible d4ffb876-f103-4eaa-9958-650f13f82a1c has session id: f834b9c0-3b16-4f68-9ffc-38e124ee79df","v":0,"name":"crucible","level":30Crucible runtime is spawned
28122024-02-03T17:23:45.806Z,"time":"2024-02-03T17:23:44.9606738Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28132024-02-03T17:23:45.806Z{"msg":"Upstairs opts: Upstairs UUID: d4ffb876-f103-4eaa-9958-650f13f82a1c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.960749041Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28142024-02-03T17:23:45.806Z{"msg":"Crucible stats registered with UUID: d4ffb876-f103-4eaa-9958-650f13f82a1c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.960803666Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28152024-02-03T17:23:45.806Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.960828377Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233}
28162024-02-03T17:23:45.806Z{"msg":"d4ffb876-f103-4eaa-9958-650f13f82a1c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.960957775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28172024-02-03T17:23:45.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961014052Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28182024-02-03T17:23:45.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961123416Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28192024-02-03T17:23:45.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961170735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28202024-02-03T17:23:45.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961244724Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28212024-02-03T17:23:45.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961348295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28222024-02-03T17:23:45.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961423981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28232024-02-03T17:23:45.807Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961667588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28242024-02-03T17:23:45.807Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961760839Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28252024-02-03T17:23:45.807Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961838987Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28262024-02-03T17:23:45.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.96189388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28272024-02-03T17:23:45.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961938981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28282024-02-03T17:23:45.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.961994101Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28292024-02-03T17:23:45.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.963123693Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28302024-02-03T17:23:45.807Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.963187448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28312024-02-03T17:23:45.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.963233286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28322024-02-03T17:23:45.807Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.963273127Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28332024-02-03T17:23:45.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.96347233Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28342024-02-03T17:23:45.807Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.963650982Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28352024-02-03T17:23:45.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.964405647Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28362024-02-03T17:23:45.811Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.964497785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28372024-02-03T17:23:45.811Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.964550667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28382024-02-03T17:23:45.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965187098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28392024-02-03T17:23:45.812Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.9652797Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28402024-02-03T17:23:45.812Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965386184Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28412024-02-03T17:23:45.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965471465Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28422024-02-03T17:23:45.812Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965544081Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28432024-02-03T17:23:45.812Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.96561435Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28442024-02-03T17:23:45.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965680849Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28452024-02-03T17:23:45.812Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965749671Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28462024-02-03T17:23:45.812Z{"msg":"[0]R flush_numbers[0..12]: [30, 39, 36, 17, 36, 40, 30, 35, 33, 36, 39, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965794484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28472024-02-03T17:23:45.812Z{"msg":"[0]R generation[0..12]: [18, 24, 24, 10, 24, 24, 18, 23, 21, 24, 24, 25]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
28482024-02-03T17:23:45.812Z,"time":"2024-02-03T17:23:44.965863839Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28492024-02-03T17:23:45.812Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965943055Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28502024-02-03T17:23:45.812Z{"msg":"[1]R flush_numbers[0..12]: [30, 39, 36, 17, 36, 40, 30, 35, 33, 36, 39, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.965990066Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28512024-02-03T17:23:45.812Z{"msg":"[1]R generation[0..12]: [18, 24, 24, 10, 24, 24, 18, 23, 21, 24, 24, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966058316Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28522024-02-03T17:23:45.812Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966130737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28532024-02-03T17:23:45.812Z{"msg":"[2]R flush_numbers[0..12]: [30, 39, 36, 17, 36, 40, 30, 35, 33, 36, 39, 41]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966204785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28542024-02-03T17:23:45.812Z{"msg":"[2]R generation[0..12]: [18, 24, 24, 10, 24, 24, 18, 23, 21, 24, 24, 25]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966276509Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28552024-02-03T17:23:45.812Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966354554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28562024-02-03T17:23:45.812Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966435548Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28572024-02-03T17:23:45.812Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966508528Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28582024-02-03T17:23:45.812Z{"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966554739Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28592024-02-03T17:23:45.812Z"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28602024-02-03T17:23:45.812Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966688849Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28612024-02-03T17:23:45.812Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966740964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28622024-02-03T17:23:45.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966763934Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28632024-02-03T17:23:45.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966804833Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28642024-02-03T17:23:45.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966828576Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28652024-02-03T17:23:45.813Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966869186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28662024-02-03T17:23:45.813Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966891345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28672024-02-03T17:23:45.813Z{"msg":"d4ffb876-f103-4eaa-9958-650f13f82a1c is now active with session: f834b9c0-3b16-4f68-9ffc-38e124ee79df","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966931571Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28682024-02-03T17:23:45.813Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.966952969Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233,"session_id":"f834b9c0-3b16-4f68-9ffc-38e124ee79df"}
28692024-02-03T17:23:45.813Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:44.96699196Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233}
28702024-02-03T17:23:45.813ZLoading write count information from file "/tmp/test_repair_verify.data"
28712024-02-03T17:23:45.813ZRun Repair workload
28722024-02-03T17:23:45.813Z01/30 Write block 405 len 40960 data: 4 4 4 4 4 4 4 4 4 3
28732024-02-03T17:23:45.817Z02/30 Read block 430 len 36864
28742024-02-03T17:23:45.817Z03/30 Write block 506 len 20480 data: 5 5 6 6 4
28752024-02-03T17:23:45.817Z04/30 Write block 69 len 4096 data: 2
28762024-02-03T17:23:45.820Z05/30 Read block 19 len 32768
28772024-02-03T17:23:45.820Z06/30 Read block 507 len 8192
28782024-02-03T17:23:45.823Z07/30 Write block 237 len 24576 data: 4 3 3 3 3 2
28792024-02-03T17:23:45.823Z08/30 Read block 516 len 40960
28802024-02-03T17:23:45.826Z09/30 Write block 497 len 32768 data: 3 3 3 3 3 3 3 2
28812024-02-03T17:23:45.826Z10/30 Flush
28822024-02-03T17:23:45.829Z11/30 Write block 207 len 20480 data: 3 2 2 4 4
28832024-02-03T17:23:45.832Z12/30 Read block 402 len 8192
28842024-02-03T17:23:45.832Z13/30 Flush
28852024-02-03T17:23:45.835Z14/30 Flush
28862024-02-03T17:23:45.835Z15/30 Write block 102 len 28672 data: 2 2 2 3 3 4 4
28872024-02-03T17:23:45.838Z16/30 Write block 421 len 20480 data: 2 2 2 2 2
28882024-02-03T17:23:45.838Z17/30 Flush
28892024-02-03T17:23:45.841Z18/30 Write block 246 len 4096 data: 2
28902024-02-03T17:23:45.841Z19/30 Read block 443 len 8192
28912024-02-03T17:23:45.844Z20/30 Write block 120 len 40960 data: 4 4 3 4 3 4 4 3 3 3
28922024-02-03T17:23:45.844Z21/30 Flush
28932024-02-03T17:23:45.847Z22/30 Write block 63 len 8192 data: 2 2
28942024-02-03T17:23:45.848Z23/30 Read block 314 len 16384
28952024-02-03T17:23:45.853Z24/30 Read block 322 len 28672
28962024-02-03T17:23:45.856Z25/30 Write block 214 len 24576 data: 4 4 2 2 2 2
28972024-02-03T17:23:45.856Z26/30 Flush
28982024-02-03T17:23:45.856Z27/30 Write block 189 len 12288 data: 4 3 3
28992024-02-03T17:23:45.859Z28/30 Read block 295 len 4096
29002024-02-03T17:23:45.860Z29/30 Read block 489 len 20480
29012024-02-03T17:23:45.860Z30/30 Read block 547 len 8192
29022024-02-03T17:23:45.863Z----------------------------------------------------------------
29032024-02-03T17:23:45.863Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29042024-02-03T17:23:45.863ZStates: Active Active Active
29052024-02-03T17:23:45.863ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29062024-02-03T17:23:45.863Z 1 Acked 1000 Write 10 Done Done Done false
29072024-02-03T17:23:45.863Z 2 Acked 1001 Read 9 Done Done Done false
29082024-02-03T17:23:45.863Z 3 Acked 1002 Write 5 Done Done Done false
29092024-02-03T17:23:45.864Z 4 Acked 1003 Write 1 Done Done Done false
29102024-02-03T17:23:45.864Z 5 Acked 1004 Read 8 Done Done Done false
29112024-02-03T17:23:45.864Z 6 Acked 1005 Read 2 Sent Done Done false
29122024-02-03T17:23:45.864Z 7 Acked 1006 Write 6 Sent Done Done false
29132024-02-03T17:23:45.864Z 8 Acked 1007 Read 10 Sent Done Done false
29142024-02-03T17:23:45.864Z 9 Acked 1008 Write 8 Sent Done Done false
29152024-02-03T17:23:45.864Z 10 Acked 1009 Flush 0 Sent Done Done false
29162024-02-03T17:23:45.864Z 11 Acked 1010 Write 5 Sent Done Done false
29172024-02-03T17:23:45.864Z 12 Acked 1011 Read 2 Sent Done Done false
29182024-02-03T17:23:45.864Z 13 Acked 1012 Flush 0 Sent Done Done false
29192024-02-03T17:23:45.864Z 14 Acked 1013 Flush 0 Sent Done Done false
29202024-02-03T17:23:45.864Z 15 Acked 1014 Write 7 Sent Done Done false
29212024-02-03T17:23:45.864Z 16 Acked 1015 Write 5 Sent Done Done false
29222024-02-03T17:23:45.864Z 17 Acked 1016 Flush 0 Sent Done Done false
29232024-02-03T17:23:45.864Z 18 Acked 1017 Write 1 Sent Done Done false
29242024-02-03T17:23:45.864Z 19 Acked 1018 Read 2 Sent Done Done false
29252024-02-03T17:23:45.864Z 20 Acked 1019 Write 10 Sent Done Done false
29262024-02-03T17:23:45.864Z 21 Acked 1020 Flush 0 Sent Done Done false
29272024-02-03T17:23:45.864Z 22 Acked 1021 Write 2 Sent Done Done false
29282024-02-03T17:23:45.864Z 23 Acked 1022 Read 4 Sent Done Done false
29292024-02-03T17:23:45.864Z 24 Acked 1023 Read 7 Sent Done Done false
29302024-02-03T17:23:45.864Z 25 Acked 1024 Write 6 Sent Done Done false
29312024-02-03T17:23:45.864Z 26 Acked 1025 Flush 0 Sent Done Done false
29322024-02-03T17:23:45.864Z 27 Acked 1026 Write 3 Sent Done Done false
29332024-02-03T17:23:45.865Z 28 Acked 1027 Read 1 Sent Done Done false
29342024-02-03T17:23:45.865Z 29 Acked 1028 Read 5 Sent Done Done false
29352024-02-03T17:23:45.865Z 30 Acked 1029 Read 2 Sent Done Sent false
29362024-02-03T17:23:45.865Z STATES DS:0 DS:1 DS:2 TOTAL
29372024-02-03T17:23:45.865Z New 0 0 0 0
29382024-02-03T17:23:45.865Z Sent 25 0 1 26
29392024-02-03T17:23:45.865Z Done 5 30 29 64
29402024-02-03T17:23:45.865Z Skipped 0 0 0 0
29412024-02-03T17:23:45.865Z Error 0 0 0 0
29422024-02-03T17:23:45.865ZLast Flush: 0 1025 1025
29432024-02-03T17:23:45.865ZDownstairs last five completed:
29442024-02-03T17:23:45.865ZUpstairs last five completed: 30 29 28 27 26
29452024-02-03T17:23:45.865Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:45.018978515Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1233}
29462024-02-03T17:23:45.865ZWrote out file "/tmp/test_repair_verify.data"
29472024-02-03T17:23:45.868Z
29482024-02-03T17:23:45.876ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29492024-02-03T17:23:45.901ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29502024-02-03T17:23:45.901Z 0 000-019 18 18 18 30 30 30 F F F
29512024-02-03T17:23:45.901Z 1 020-039 24 24 24 39 39 39 F F F
29522024-02-03T17:23:45.901Z 2 040-059 24 24 24 36 36 36 F F F
29532024-02-03T17:23:45.905Z 3 060-079 10 26 26 17 47 47 T F F <---
29542024-02-03T17:23:45.905Z 4 080-099 24 24 24 36 36 36 F F F
29552024-02-03T17:23:45.905Z 5 100-119 24 26 26 40 45 45 F F F <---
29562024-02-03T17:23:45.905Z 6 120-139 18 26 26 30 46 46 F F F <---
29572024-02-03T17:23:45.905Z 7 140-159 23 23 23 35 35 35 F F F
29582024-02-03T17:23:45.905Z 8 160-179 21 21 21 33 33 33 F F F
29592024-02-03T17:23:45.905Z 9 180-199 24 24 24 36 36 36 F T T <---
29602024-02-03T17:23:45.905Z 10 200-219 24 26 26 39 47 47 F F F <---
29612024-02-03T17:23:45.905Z 11 220-239 25 26 26 41 42 42 F F F <---
29622024-02-03T17:23:45.905Z 12 240-259 23 26 26 35 46 46 F F F <---
29632024-02-03T17:23:45.905Z 13 260-279 25 25 25 41 41 41 F F F
29642024-02-03T17:23:45.905Z 14 280-299 25 25 25 41 41 41 F F F
29652024-02-03T17:23:45.905Z 15 300-319 14 14 14 23 23 23 F F F
29662024-02-03T17:23:45.905Z 16 320-339 23 23 23 35 35 35 F F F
29672024-02-03T17:23:45.905Z 17 340-359 24 24 24 40 40 40 F F F
29682024-02-03T17:23:45.905Z 18 360-379 23 23 23 35 35 35 F F F
29692024-02-03T17:23:45.906Z 19 380-399 19 19 19 31 31 31 F F F
29702024-02-03T17:23:45.906Z 20 400-419 22 26 26 34 42 42 T F F <---
29712024-02-03T17:23:45.906Z 21 420-439 16 26 26 27 45 45 F F F <---
29722024-02-03T17:23:45.906Z 22 440-459 24 24 24 37 37 37 F F F
29732024-02-03T17:23:45.906Z 23 460-479 24 24 24 37 37 37 F F F
29742024-02-03T17:23:45.906Z 24 480-499 13 26 26 22 42 42 F F F <---
29752024-02-03T17:23:45.906Z 25 500-519 24 26 26 36 42 42 T F F <---
29762024-02-03T17:23:45.906Z 26 520-539 25 25 25 41 41 41 F F F
29772024-02-03T17:23:45.906Z 27 540-559 23 23 23 35 35 35 F F F
29782024-02-03T17:23:45.906Z 28 560-579 24 24 24 40 40 40 F F F
29792024-02-03T17:23:45.906Z 29 580-599 23 23 23 35 35 35 F F F
29802024-02-03T17:23:45.906ZMax gen: 26, Max flush: 47
29812024-02-03T17:23:45.906ZError: Difference in extent metadata found!
29822024-02-03T17:23:45.909ZA Difference in extent metadata is expected here
29832024-02-03T17:23:45.909ZOn loop 13
29842024-02-03T17:23:45.909Z
29852024-02-03T17:23:45.909ZVerifying data now
29862024-02-03T17:23:51.898ZNo change after verify
29872024-02-03T17:23:51.898ZLoop: 13 Downstairs dump after verify (and repair):
29882024-02-03T17:23:51.923ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29892024-02-03T17:23:51.923Z 0 000-019 18 18 18 30 30 30 F F F
29902024-02-03T17:23:51.923Z 1 020-039 24 24 24 39 39 39 F F F
29912024-02-03T17:23:51.923Z 2 040-059 24 24 24 36 36 36 F F F
29922024-02-03T17:23:51.923Z 3 060-079 26 26 26 47 47 47 F F F
29932024-02-03T17:23:51.923Z 4 080-099 24 24 24 36 36 36 F F F
29942024-02-03T17:23:51.923Z 5 100-119 26 26 26 45 45 45 F F F
29952024-02-03T17:23:51.923Z 6 120-139 26 26 26 46 46 46 F F F
29962024-02-03T17:23:51.923Z 7 140-159 23 23 23 35 35 35 F F F
29972024-02-03T17:23:51.923Z 8 160-179 21 21 21 33 33 33 F F F
29982024-02-03T17:23:51.923Z 9 180-199 27 27 27 48 48 48 F F F
29992024-02-03T17:23:51.923Z 10 200-219 26 26 26 47 47 47 F F F
30002024-02-03T17:23:51.928Z 11 220-239 26 26 26 42 42 42 F F F
30012024-02-03T17:23:51.928Z 12 240-259 26 26 26 46 46 46 F F F
30022024-02-03T17:23:51.928Z 13 260-279 25 25 25 41 41 41 F F F
30032024-02-03T17:23:51.928Z 14 280-299 25 25 25 41 41 41 F F F
30042024-02-03T17:23:51.928Z 15 300-319 14 14 14 23 23 23 F F F
30052024-02-03T17:23:51.928Z 16 320-339 23 23 23 35 35 35 F F F
30062024-02-03T17:23:51.929Z 17 340-359 24 24 24 40 40 40 F F F
30072024-02-03T17:23:51.929Z 18 360-379 23 23 23 35 35 35 F F F
30082024-02-03T17:23:51.929Z 19 380-399 19 19 19 31 31 31 F F F
30092024-02-03T17:23:51.929Z 20 400-419 26 26 26 42 42 42 F F F
30102024-02-03T17:23:51.929Z 21 420-439 26 26 26 45 45 45 F F F
30112024-02-03T17:23:51.929Z 22 440-459 24 24 24 37 37 37 F F F
30122024-02-03T17:23:51.929Z 23 460-479 24 24 24 37 37 37 F F F
30132024-02-03T17:23:51.929Z 24 480-499 26 26 26 42 42 42 F F F
30142024-02-03T17:23:51.929Z 25 500-519 26 26 26 42 42 42 F F F
30152024-02-03T17:23:51.929Z 26 520-539 25 25 25 41 41 41 F F F
30162024-02-03T17:23:51.929Z 27 540-559 23 23 23 35 35 35 F F F
30172024-02-03T17:23:51.929Z 28 560-579 24 24 24 40 40 40 F F F
30182024-02-03T17:23:51.930Z 29 580-599 23 23 23 35 35 35 F F F
30192024-02-03T17:23:51.930ZMax gen: 27, Max flush: 48
30202024-02-03T17:23:51.930Z
30212024-02-03T17:23:51.930ZBegin loop 14
30222024-02-03T17:23:51.930ZDownstairs to restart: 0
30232024-02-03T17:23:51.960Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.117936145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241}
30242024-02-03T17:23:51.960Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.118448543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241}
30252024-02-03T17:23:51.964Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.118531331Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241}
30262024-02-03T17:23:51.964Z{"msg":"Crucible c1972758-d271-426f-b254-2bc181511c45 has session id: 13125551-aebf-4dc9-87af-b784d37a1f7b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.118599591Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"Crucible runtime is spawned
30272024-02-03T17:23:51.964Z13125551-aebf-4dc9-87af-b784d37a1f7b"}
30282024-02-03T17:23:51.964Z{"msg":"Upstairs opts: Upstairs UUID: c1972758-d271-426f-b254-2bc181511c45, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.118762673Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30292024-02-03T17:23:51.964Z{"msg":"Crucible stats registered with UUID: c1972758-d271-426f-b254-2bc181511c45","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.118838615Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30302024-02-03T17:23:51.964Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.118907301Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241}
30312024-02-03T17:23:51.965Z{"msg":"c1972758-d271-426f-b254-2bc181511c45 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.118944341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30322024-02-03T17:23:51.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119013419Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30332024-02-03T17:23:51.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119058492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30342024-02-03T17:23:51.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.11915038Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30352024-02-03T17:23:51.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119197756Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30362024-02-03T17:23:51.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119229632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30372024-02-03T17:23:51.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.11925644Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30382024-02-03T17:23:51.965Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119326761Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30392024-02-03T17:23:51.965Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119432937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30402024-02-03T17:23:51.965Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119574933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30412024-02-03T17:23:51.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119630561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30422024-02-03T17:23:51.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119674325Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30432024-02-03T17:23:51.965Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.119744691Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30442024-02-03T17:23:51.965Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.121000293Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30452024-02-03T17:23:51.965Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.121168472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30462024-02-03T17:23:51.965Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.12141414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30472024-02-03T17:23:51.965Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.121491699Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30482024-02-03T17:23:51.970Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.122015633Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30492024-02-03T17:23:51.970Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.122096627Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30502024-02-03T17:23:51.970Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.122126606Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30512024-02-03T17:23:51.970Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.122190377Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30522024-02-03T17:23:51.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.122987468Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30532024-02-03T17:23:51.970Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.123076363Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30542024-02-03T17:23:51.971Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.123124742Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30552024-02-03T17:23:51.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.123199544Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30562024-02-03T17:23:51.971Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.123276514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30572024-02-03T17:23:51.971Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.123349658Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30582024-02-03T17:23:51.971Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.123392495Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30592024-02-03T17:23:51.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.12420479Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30602024-02-03T17:23:51.971Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124290359Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30612024-02-03T17:23:51.971Z{"msg":"[0]R flush_numbers[0..12]: [30, 39, 36, 47, 36, 45, 46, 35, 33, 48, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124364591Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30622024-02-03T17:23:51.971Z{"msg":"[0]R generation[0..12]: [18, 24, 24, 26, 24, 26, 26, 23, 21, 27, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124437864Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30632024-02-03T17:23:51.971Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124509038Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30642024-02-03T17:23:51.971Z{"msg":"[1]R flush_numbers[0..12]: [30, 39, 36, 47, 36, 45, 46, 35, 33, 48, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124582634Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30652024-02-03T17:23:51.971Z{"msg"Wait for a query_work_queue command to finish before sending IO
30662024-02-03T17:23:51.971Z:"[1]R generation[0..12]: [18, 24, 24, 26, 24, 26, 26, 23, 21, 27, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124668085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30672024-02-03T17:23:51.972Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124714238Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30682024-02-03T17:23:51.972Z{"msg":"[2]R flush_numbers[0..12]: [30, 39, 36, 47, 36, 45, 46, 35, 33, 48, 47, 42]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124788992Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30692024-02-03T17:23:51.972Z{"msg":"[2]R generation[0..12]: [18, 24, 24, 26, 24, 26, 26, 23, 21, 27, 26, 26]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.124863767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30702024-02-03T17:23:51.972Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.12493548Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30712024-02-03T17:23:51.972Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125010011Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30722024-02-03T17:23:51.972Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125056437Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30732024-02-03T17:23:51.972Z{"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125126672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30742024-02-03T17:23:51.972Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125173744Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30752024-02-03T17:23:51.972Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125245573Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30762024-02-03T17:23:51.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125272447Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30772024-02-03T17:23:51.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125314515Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30782024-02-03T17:23:51.972Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125338074Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30792024-02-03T17:23:51.972Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125387179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30802024-02-03T17:23:51.973Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125409441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30812024-02-03T17:23:51.973Z{"msg":"c1972758-d271-426f-b254-2bc181511c45 is now active with session: 13125551-aebf-4dc9-87af-b784d37a1f7b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125430672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30822024-02-03T17:23:51.973Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125450206Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241,"session_id":"13125551-aebf-4dc9-87af-b784d37a1f7b"}
30832024-02-03T17:23:51.973Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:51.125497634Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241}
30842024-02-03T17:23:51.973ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30852024-02-03T17:23:51.973ZLoading write count information from file "/tmp/test_repair_verify.data"
30862024-02-03T17:23:51.973ZRun Repair workload
30872024-02-03T17:23:51.973Z01/30 Write block 217 len 24576 data: 3 3 3 2 5 5
30882024-02-03T17:23:51.977Z02/30 Read block 585 len 36864
30892024-02-03T17:23:51.977Z03/30 Write block 276 len 8192 data: 6 5
30902024-02-03T17:23:51.977Z04/30 Write block 462 len 40960 data: 4 4 4 5 3 3 4 5 4 4
30912024-02-03T17:23:51.977Z05/30 Write block 501 len 24576 data: 4 4 4 3 4 6
30922024-02-03T17:23:51.977Z06/30 Flush
30932024-02-03T17:23:51.981Z07/30 Write block 322 len 28672 data: 2 2 2 3 3 3 5
30942024-02-03T17:23:51.984Z08/30 Read block 485 len 4096
30952024-02-03T17:23:51.984Z09/30 Write block 161 len 4096 data: 3
30962024-02-03T17:23:51.984Z10/30 Flush
30972024-02-03T17:23:51.988Z11/30 Write block 409 len 24576 data: 5 5 5 5 5 4
30982024-02-03T17:23:51.991Z12/30 Write block 215 len 24576 data: 5 3 4 4 4 3
30992024-02-03T17:23:51.991Z13/30 Read block 421 len 4096
31002024-02-03T17:23:51.991Z14/30 Read block 573 len 40960
31012024-02-03T17:23:51.995Z15/30 Write block 162 len 24576 data: 4 3 3 3 3 4
31022024-02-03T17:23:51.995Z16/30 Read block 93 len 20480
31032024-02-03T17:23:51.995Z17/30 Read block 265 len 40960
31042024-02-03T17:23:51.999Z18/30 Write block 275 len 8192 data: 7 7
31052024-02-03T17:23:51.999Z19/30 Write block 239 len 4096 data: 4
31062024-02-03T17:23:52.003Z20/30 Write block 89 len 20480 data: 3 3 2 2 3
31072024-02-03T17:23:52.004Z21/30 Read block 194 len 16384
31082024-02-03T17:23:52.004Z22/30 Read block 339 len 20480
31092024-02-03T17:23:52.004Z23/30 Read block 519 len 8192
31102024-02-03T17:23:52.012Z24/30 Write block 531 len 36864 data: 5 4 3 2 3 3 3 3 3
31112024-02-03T17:23:52.012Z25/30 Read block 516 len 12288
31122024-02-03T17:23:52.012Z26/30 Flush
31132024-02-03T17:23:52.016Z27/30 Write block 456 len 4096 data: 3
31142024-02-03T17:23:52.016Z28/30 Read block 343 len 16384
31152024-02-03T17:23:52.016Z29/30 Write block 380 len 40960 data: 3 3 2 3 3 3 3 3 3 3
31162024-02-03T17:23:52.016Z30/30 Read block 320 len 8192
31172024-02-03T17:23:52.016Z----------------------------------------------------------------
31182024-02-03T17:23:52.020Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31192024-02-03T17:23:52.020ZStates: Active Active Active
31202024-02-03T17:23:52.020ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31212024-02-03T17:23:52.020Z 1 Acked 1000 Write 6 Done Done Done false
31222024-02-03T17:23:52.020Z 2 Acked 1001 Read 9 Done Done Done false
31232024-02-03T17:23:52.020Z 3 Acked 1002 Write 2 Sent Done Done false
31242024-02-03T17:23:52.020Z 4 Acked 1003 Write 10 Sent Done Done false
31252024-02-03T17:23:52.021Z 5 Acked 1004 Write 6 Sent Done Done false
31262024-02-03T17:23:52.021Z 6 Acked 1005 Flush 0 Sent Done Done false
31272024-02-03T17:23:52.021Z 7 Acked 1006 Write 7 Sent Done Done false
31282024-02-03T17:23:52.021Z 8 Acked 1007 Read 1 Sent Done Done false
31292024-02-03T17:23:52.021Z 9 Acked 1008 Write 1 Sent Done Done false
31302024-02-03T17:23:52.021Z 10 Acked 1009 Flush 0 Sent Done Done false
31312024-02-03T17:23:52.021Z 11 Acked 1010 Write 6 Sent Done Done false
31322024-02-03T17:23:52.021Z 12 Acked 1011 Write 6 Sent Done Done false
31332024-02-03T17:23:52.021Z 13 Acked 1012 Read 1 Sent Done Done false
31342024-02-03T17:23:52.021Z 14 Acked 1013 Read 10 Sent Done Done false
31352024-02-03T17:23:52.021Z 15 Acked 1014 Write 6 Sent Done Done false
31362024-02-03T17:23:52.021Z 16 Acked 1015 Read 5 Sent Done Done false
31372024-02-03T17:23:52.021Z 17 Acked 1016 Read 10 Sent Done Done false
31382024-02-03T17:23:52.021Z 18 Acked 1017 Write 2 Sent Done Done false
31392024-02-03T17:23:52.022Z 19 Acked 1018 Write 1 Sent Done Done false
31402024-02-03T17:23:52.022Z 20 Acked 1019 Write 5 Sent Done Done false
31412024-02-03T17:23:52.022Z 21 Acked 1020 Read 4 Sent Done Done false
31422024-02-03T17:23:52.022Z 22 Acked 1021 Read 5 Sent Done Done false
31432024-02-03T17:23:52.022Z 23 Acked 1022 Read 2 Sent Done Done false
31442024-02-03T17:23:52.022Z 24 Acked 1023 Write 9 Sent Done Done false
31452024-02-03T17:23:52.022Z 25 Acked 1024 Read 3 Sent Done Done false
31462024-02-03T17:23:52.022Z 26 Acked 1025 Flush 0 Sent Done Done false
31472024-02-03T17:23:52.022Z 27 Acked 1026 Write 1 Sent Done Done false
31482024-02-03T17:23:52.022Z 28 Acked 1027 Read 4 Sent Done Done false
31492024-02-03T17:23:52.022Z 29 Acked 1028 Write 10 Sent Sent Done false
31502024-02-03T17:23:52.022Z 30 Acked 1029 Read 2 Sent Sent Done false
31512024-02-03T17:23:52.022Z STATES DS:0 DS:1 DS:2 TOTAL
31522024-02-03T17:23:52.023Z New 0 0 0 0
31532024-02-03T17:23:52.023Z Sent 28 2 0 30
31542024-02-03T17:23:52.023Z Done 2 28 30 60
31552024-02-03T17:23:52.023Z Skipped 0 0 0 0
31562024-02-03T17:23:52.023Z Error 0 0 0 0
31572024-02-03T17:23:52.023ZLast Flush: 0 1025 1025
31582024-02-03T17:23:52.023ZDownstairs last five completed:
31592024-02-03T17:23:52.023ZUpstairs last five completed: 30 29 28 27 26
31602024-02-03T17:23:52.023Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:51.17495069Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1241}
31612024-02-03T17:23:52.023ZWrote out file "/tmp/test_repair_verify.data"
31622024-02-03T17:23:52.027Z
31632024-02-03T17:23:52.032ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31642024-02-03T17:23:52.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31652024-02-03T17:23:52.057Z 0 000-019 18 18 18 30 30 30 F F F
31662024-02-03T17:23:52.057Z 1 020-039 24 24 24 39 39 39 F F F
31672024-02-03T17:23:52.057Z 2 040-059 24 24 24 36 36 36 F F F
31682024-02-03T17:23:52.057Z 3 060-079 26 26 26 47 47 47 F F F
31692024-02-03T17:23:52.057Z 4 080-099 24 28 28 36 51 51 F F F <---
31702024-02-03T17:23:52.057Z 5 100-119 26 26 26 45 45 45 F F F
31712024-02-03T17:23:52.057Z 6 120-139 26 26 26 46 46 46 F F F
31722024-02-03T17:23:52.061Z 7 140-159 23 23 23 35 35 35 F F F
31732024-02-03T17:23:52.061Z 8 160-179 21 28 28 33 51 51 F F F <---
31742024-02-03T17:23:52.061Z 9 180-199 27 27 27 48 48 48 F F F
31752024-02-03T17:23:52.061Z 10 200-219 26 28 28 47 51 51 T F F <---
31762024-02-03T17:23:52.061Z 11 220-239 26 28 28 42 51 51 T F F <---
31772024-02-03T17:23:52.061Z 12 240-259 26 26 26 46 46 46 F F F
31782024-02-03T17:23:52.061Z 13 260-279 25 28 28 41 51 51 F F F <---
31792024-02-03T17:23:52.061Z 14 280-299 25 25 25 41 41 41 F F F
31802024-02-03T17:23:52.061Z 15 300-319 14 14 14 23 23 23 F F F
31812024-02-03T17:23:52.061Z 16 320-339 23 28 28 35 50 50 F F F <---
31822024-02-03T17:23:52.061Z 17 340-359 24 24 24 40 40 40 F F F
31832024-02-03T17:23:52.061Z 18 360-379 23 23 23 35 35 35 F F F
31842024-02-03T17:23:52.061Z 19 380-399 19 19 19 31 31 31 F T T <---
31852024-02-03T17:23:52.061Z 20 400-419 26 28 28 42 51 51 F F F <---
31862024-02-03T17:23:52.061Z 21 420-439 26 26 26 45 45 45 F F F
31872024-02-03T17:23:52.061Z 22 440-459 24 24 24 37 37 37 F T T <---
31882024-02-03T17:23:52.061Z 23 460-479 24 28 28 37 49 49 F F F <---
31892024-02-03T17:23:52.061Z 24 480-499 26 26 26 42 42 42 F F F
31902024-02-03T17:23:52.061Z 25 500-519 26 28 28 42 49 49 F F F <---
31912024-02-03T17:23:52.061Z 26 520-539 25 28 28 41 51 51 F F F <---
31922024-02-03T17:23:52.061Z 27 540-559 23 23 23 35 35 35 F F F
31932024-02-03T17:23:52.062Z 28 560-579 24 24 24 40 40 40 F F F
31942024-02-03T17:23:52.062Z 29 580-599 23 23 23 35 35 35 F F F
31952024-02-03T17:23:52.062ZMax gen: 28, Max flush: 51
31962024-02-03T17:23:52.062ZError: Difference in extent metadata found!
31972024-02-03T17:23:52.065ZA Difference in extent metadata is expected here
31982024-02-03T17:23:52.065ZOn loop 14
31992024-02-03T17:23:52.065Z
32002024-02-03T17:23:52.068ZVerifying data now
32012024-02-03T17:23:58.136ZNo change after verify
32022024-02-03T17:23:58.136ZLoop: 14 Downstairs dump after verify (and repair):
32032024-02-03T17:23:58.160ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32042024-02-03T17:23:58.160Z 0 000-019 18 18 18 30 30 30 F F F
32052024-02-03T17:23:58.164Z 1 020-039 24 24 24 39 39 39 F F F
32062024-02-03T17:23:58.164Z 2 040-059 24 24 24 36 36 36 F F F
32072024-02-03T17:23:58.164Z 3 060-079 26 26 26 47 47 47 F F F
32082024-02-03T17:23:58.164Z 4 080-099 28 28 28 51 51 51 F F F
32092024-02-03T17:23:58.164Z 5 100-119 26 26 26 45 45 45 F F F
32102024-02-03T17:23:58.164Z 6 120-139 26 26 26 46 46 46 F F F
32112024-02-03T17:23:58.164Z 7 140-159 23 23 23 35 35 35 F F F
32122024-02-03T17:23:58.164Z 8 160-179 28 28 28 51 51 51 F F F
32132024-02-03T17:23:58.164Z 9 180-199 27 27 27 48 48 48 F F F
32142024-02-03T17:23:58.164Z 10 200-219 28 28 28 51 51 51 F F F
32152024-02-03T17:23:58.164Z 11 220-239 28 28 28 51 51 51 F F F
32162024-02-03T17:23:58.165Z 12 240-259 26 26 26 46 46 46 F F F
32172024-02-03T17:23:58.165Z 13 260-279 28 28 28 51 51 51 F F F
32182024-02-03T17:23:58.165Z 14 280-299 25 25 25 41 41 41 F F F
32192024-02-03T17:23:58.165Z 15 300-319 14 14 14 23 23 23 F F F
32202024-02-03T17:23:58.165Z 16 320-339 28 28 28 50 50 50 F F F
32212024-02-03T17:23:58.165Z 17 340-359 24 24 24 40 40 40 F F F
32222024-02-03T17:23:58.165Z 18 360-379 23 23 23 35 35 35 F F F
32232024-02-03T17:23:58.165Z 19 380-399 29 29 29 52 52 52 F F F
32242024-02-03T17:23:58.165Z 20 400-419 28 28 28 51 51 51 F F F
32252024-02-03T17:23:58.165Z 21 420-439 26 26 26 45 45 45 F F F
32262024-02-03T17:23:58.165Z 22 440-459 29 29 29 52 52 52 F F F
32272024-02-03T17:23:58.165Z 23 460-479 28 28 28 49 49 49 F F F
32282024-02-03T17:23:58.165Z 24 480-499 26 26 26 42 42 42 F F F
32292024-02-03T17:23:58.165Z 25 500-519 28 28 28 49 49 49 F F F
32302024-02-03T17:23:58.166Z 26 520-539 28 28 28 51 51 51 F F F
32312024-02-03T17:23:58.166Z 27 540-559 23 23 23 35 35 35 F F F
32322024-02-03T17:23:58.166Z 28 560-579 24 24 24 40 40 40 F F F
32332024-02-03T17:23:58.166Z 29 580-599 23 23 23 35 35 35 F F F
32342024-02-03T17:23:58.166ZMax gen: 29, Max flush: 52
32352024-02-03T17:23:58.180Z
32362024-02-03T17:23:58.180ZBegin loop 15
32372024-02-03T17:23:58.180ZDownstairs to restart: 2
32382024-02-03T17:23:58.197Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.355576922Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249}
32392024-02-03T17:23:58.201Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356048804Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249}
32402024-02-03T17:23:58.201Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356123821Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249}
32412024-02-03T17:23:58.201Z{"msg":"Crucible runtime is spawned
32422024-02-03T17:23:58.201ZCrucible 2d203f35-0d3b-491c-ab00-d96453381af6 has session id: d2e9f754-6d23-48ff-b1d1-23be86804a99","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356206418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32432024-02-03T17:23:58.201Z{"msg":"Upstairs opts: Upstairs UUID: 2d203f35-0d3b-491c-ab00-d96453381af6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356319173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32442024-02-03T17:23:58.202Z{"msg":"Crucible stats registered with UUID: 2d203f35-0d3b-491c-ab00-d96453381af6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356365305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32452024-02-03T17:23:58.202Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356429566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249}
32462024-02-03T17:23:58.202Z{"msg":"2d203f35-0d3b-491c-ab00-d96453381af6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356454526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32472024-02-03T17:23:58.202Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356496364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32482024-02-03T17:23:58.202Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356522369Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32492024-02-03T17:23:58.202Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356545236Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32502024-02-03T17:23:58.202Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356640921Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32512024-02-03T17:23:58.202Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356687986Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32522024-02-03T17:23:58.202Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356727708Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32532024-02-03T17:23:58.202Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356813813Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32542024-02-03T17:23:58.202Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356910933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32552024-02-03T17:23:58.202Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.356986421Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32562024-02-03T17:23:58.202Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.357078822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32572024-02-03T17:23:58.203Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.357150987Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32582024-02-03T17:23:58.203Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.357366125Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32592024-02-03T17:23:58.203Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.358310516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32602024-02-03T17:23:58.203Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.358439668Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32612024-02-03T17:23:58.203Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.358580818Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32622024-02-03T17:23:58.203Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.358632425Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32632024-02-03T17:23:58.203Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.359331288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32642024-02-03T17:23:58.203Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.359406714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32652024-02-03T17:23:58.208Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.359687764Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32662024-02-03T17:23:58.208Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.359808678Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32672024-02-03T17:23:58.208Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.360507857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32682024-02-03T17:23:58.208Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.360567703Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32692024-02-03T17:23:58.208Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.360593683Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32702024-02-03T17:23:58.208Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.360655663Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32712024-02-03T17:23:58.208Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.360715033Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32722024-02-03T17:23:58.208Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.360739694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32732024-02-03T17:23:58.209Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.361043345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32742024-02-03T17:23:58.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.361730707Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32752024-02-03T17:23:58.209Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.361816047Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32762024-02-03T17:23:58.209Z{"msg":"[0]R flush_numbers[0..12]: [30, 39, 36, 47, 51, 45, 46, 35, 51, 48, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.361852281Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32772024-02-03T17:23:58.209Z{"msg":"[0]R generation[0..12]: [18, 24, 24, 26, 28, 26, 26, 23, 28, 27, 28, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.361877274Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32782024-02-03T17:23:58.209Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.361902759Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32792024-02-03T17:23:58.209Z{"msg":"[1]R flush_numbers[0..12]: [30, 39, 36, 47, 51, 45, 46, 35, 51, 48, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.361964721Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32802024-02-03T17:23:58.209Z{"msg":"[1]R generation[0..12]: [18, 24, 24, 26, 28, 26, 26, 23, 28, 27, 28, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.361996827Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32812024-02-03T17:23:58.209Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.3620434Z","hostname":"Wait for a query_work_queue command to finish before sending IO
32822024-02-03T17:23:58.209Zip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32832024-02-03T17:23:58.209Z{"msg":"[2]R flush_numbers[0..12]: [30, 39, 36, 47, 51, 45, 46, 35, 51, 48, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362075229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32842024-02-03T17:23:58.209Z{"msg":"[2]R generation[0..12]: [18, 24, 24, 26, 28, 26, 26, 23, 28, 27, 28, 28]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362123969Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32852024-02-03T17:23:58.209Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362150354Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32862024-02-03T17:23:58.210Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362172826Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32872024-02-03T17:23:58.210Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362236271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32882024-02-03T17:23:58.210Z{"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362272424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32892024-02-03T17:23:58.210Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362359935Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32902024-02-03T17:23:58.210Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362405312Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32912024-02-03T17:23:58.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362443159Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32922024-02-03T17:23:58.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362510295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32932024-02-03T17:23:58.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362549477Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32942024-02-03T17:23:58.210Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362646269Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32952024-02-03T17:23:58.210Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362712623Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32962024-02-03T17:23:58.210Z{"msg":"2d203f35-0d3b-491c-ab00-d96453381af6 is now active with session: d2e9f754-6d23-48ff-b1d1-23be86804a99","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.36275624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32972024-02-03T17:23:58.210Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362841022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249,"session_id":"d2e9f754-6d23-48ff-b1d1-23be86804a99"}
32982024-02-03T17:23:58.211Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:23:57.362884956Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249}
32992024-02-03T17:23:58.211ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33002024-02-03T17:23:58.211ZLoading write count information from file "/tmp/test_repair_verify.data"
33012024-02-03T17:23:58.211ZRun Repair workload
33022024-02-03T17:23:58.211Z01/30 Write block 186 len 8192 data: 3 4
33032024-02-03T17:23:58.215Z02/30 Flush
33042024-02-03T17:23:58.215Z03/30 Write block 276 len 4096 data: 8
33052024-02-03T17:23:58.215Z04/30 Write block 417 len 16384 data: 3 2 2 2
33062024-02-03T17:23:58.216Z05/30 Flush
33072024-02-03T17:23:58.216Z06/30 Write block 340 len 16384 data: 5 4 4 4
33082024-02-03T17:23:58.219Z07/30 Read block 338 len 24576
33092024-02-03T17:23:58.219Z08/30 Write block 145 len 4096 data: 2
33102024-02-03T17:23:58.219Z09/30 Write block 359 len 12288 data: 3 4 4
33112024-02-03T17:23:58.219Z10/30 Read block 464 len 20480
33122024-02-03T17:23:58.224Z11/30 Read block 462 len 4096
33132024-02-03T17:23:58.225Z12/30 Write block 221 len 8192 data: 6 6
33142024-02-03T17:23:58.225Z13/30 Read block 28 len 40960
33152024-02-03T17:23:58.228Z14/30 Read block 234 len 20480
33162024-02-03T17:23:58.228Z15/30 Read block 581 len 20480
33172024-02-03T17:23:58.228Z16/30 Read block 315 len 16384
33182024-02-03T17:23:58.231Z17/30 Write block 225 len 40960 data: 5 4 5 4 2 2 2 2 2 3
33192024-02-03T17:23:58.231Z18/30 Write block 586 len 16384 data: 3 2 2 3
33202024-02-03T17:23:58.234Z19/30 Write block 21 len 36864 data: 4 4 4 4 4 2 2 3 3
33212024-02-03T17:23:58.234Z20/30 Write block 453 len 28672 data: 4 3 2 4 4 4 4
33222024-02-03T17:23:58.234Z21/30 Read block 129 len 8192
33232024-02-03T17:23:58.240Z22/30 Read block 238 len 28672
33242024-02-03T17:23:58.240Z23/30 Write block 236 len 16384 data: 4 5 4 5
33252024-02-03T17:23:58.240Z24/30 Read block 559 len 4096
33262024-02-03T17:23:58.240Z25/30 Write block 591 len 20480 data: 5 5 5 5 5
33272024-02-03T17:23:58.243Z26/30 Write block 580 len 4096 data: 5
33282024-02-03T17:23:58.243Z27/30 Write block 87 len 36864 data: 4 3 4 4 3 3 4 3 4
33292024-02-03T17:23:58.243Z28/30 Write block 467 len 16384 data: 4 5 6 5
33302024-02-03T17:23:58.246Z29/30 Write block 134 len 24576 data: 3 4 5 5 5 5
33312024-02-03T17:23:58.247Z30/30 Read block 48 len 32768
33322024-02-03T17:23:58.250Z----------------------------------------------------------------
33332024-02-03T17:23:58.250Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33342024-02-03T17:23:58.251ZStates: Active Active Active
33352024-02-03T17:23:58.251ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33362024-02-03T17:23:58.251Z 1 Acked 1000 Write 2 Done Done Sent false
33372024-02-03T17:23:58.251Z 2 Acked 1001 Flush 0 Done Done Sent false
33382024-02-03T17:23:58.251Z 3 Acked 1002 Write 1 Done Done Sent false
33392024-02-03T17:23:58.251Z 4 Acked 1003 Write 4 Done Done Sent false
33402024-02-03T17:23:58.251Z 5 Acked 1004 Flush 0 Done Done Sent false
33412024-02-03T17:23:58.251Z 6 Acked 1005 Write 4 Done Done Sent false
33422024-02-03T17:23:58.251Z 7 Acked 1006 Read 6 Done Done Sent false
33432024-02-03T17:23:58.251Z 8 Acked 1007 Write 1 Done Done Sent false
33442024-02-03T17:23:58.251Z 9 Acked 1008 Write 3 Done Done Sent false
33452024-02-03T17:23:58.251Z 10 Acked 1009 Read 5 Done Done Sent false
33462024-02-03T17:23:58.252Z 11 Acked 1010 Read 1 Done Done Sent false
33472024-02-03T17:23:58.252Z 12 Acked 1011 Write 2 Done Done Sent false
33482024-02-03T17:23:58.252Z 13 Acked 1012 Read 10 Done Done Sent false
33492024-02-03T17:23:58.252Z 14 Acked 1013 Read 5 Done Done Sent false
33502024-02-03T17:23:58.252Z 15 Acked 1014 Read 5 Done Done Sent false
33512024-02-03T17:23:58.252Z 16 Acked 1015 Read 4 Done Done Sent false
33522024-02-03T17:23:58.252Z 17 Acked 1016 Write 10 Done Done Sent false
33532024-02-03T17:23:58.252Z 18 Acked 1017 Write 4 Done Done Sent false
33542024-02-03T17:23:58.252Z 19 Acked 1018 Write 9 Done Done Sent false
33552024-02-03T17:23:58.252Z 20 Acked 1019 Write 7 Done Done Sent false
33562024-02-03T17:23:58.252Z 21 Acked 1020 Read 2 Done Done Sent false
33572024-02-03T17:23:58.252Z 22 Acked 1021 Read 7 Done Done Sent false
33582024-02-03T17:23:58.252Z 23 Acked 1022 Write 4 Done Done Sent false
33592024-02-03T17:23:58.253Z 24 Acked 1023 Read 1 Done Done Sent false
33602024-02-03T17:23:58.253Z 25 Acked 1024 Write 5 Done Done Sent false
33612024-02-03T17:23:58.253Z 26 Acked 1025 Write 1 Done Done Sent false
33622024-02-03T17:23:58.253Z 27 Acked 1026 Write 9 Done Done Sent false
33632024-02-03T17:23:58.253Z 28 Acked 1027 Write 4 Done Done Sent false
33642024-02-03T17:23:58.253Z 29 Acked 1028 Write 6 Done Done Sent false
33652024-02-03T17:23:58.253Z 30 Acked 1029 Read 8 Done Sent Sent false
33662024-02-03T17:23:58.253Z STATES DS:0 DS:1 DS:2 TOTAL
33672024-02-03T17:23:58.253Z New 0 0 0 0
33682024-02-03T17:23:58.253Z Sent 0 1 30 31
33692024-02-03T17:23:58.253Z Done 30 29 0 59
33702024-02-03T17:23:58.253Z Skipped 0 0 0 0
33712024-02-03T17:23:58.253Z Error 0 0 0 0
33722024-02-03T17:23:58.254ZLast Flush: 1004 1004 0
33732024-02-03T17:23:58.254ZDownstairs last five completed:
33742024-02-03T17:23:58.254ZUpstairs last five completed: 30 29 28 27 26
33752024-02-03T17:23:58.254Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:23:57.406243313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1249}
33762024-02-03T17:23:58.254ZWrote out file "/tmp/test_repair_verify.data"
33772024-02-03T17:23:58.257Z
33782024-02-03T17:23:58.261ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33792024-02-03T17:23:58.290ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33802024-02-03T17:23:58.290Z 0 000-019 18 18 18 30 30 30 F F F
33812024-02-03T17:23:58.293Z 1 020-039 24 24 24 39 39 39 T T F <---
33822024-02-03T17:23:58.293Z 2 040-059 24 24 24 36 36 36 F F F
33832024-02-03T17:23:58.293Z 3 060-079 26 26 26 47 47 47 F F F
33842024-02-03T17:23:58.294Z 4 080-099 28 28 28 51 51 51 T T F <---
33852024-02-03T17:23:58.294Z 5 100-119 26 26 26 45 45 45 F F F
33862024-02-03T17:23:58.294Z 6 120-139 26 26 26 46 46 46 T T F <---
33872024-02-03T17:23:58.294Z 7 140-159 23 23 23 35 35 35 T T F <---
33882024-02-03T17:23:58.294Z 8 160-179 28 28 28 51 51 51 F F F
33892024-02-03T17:23:58.294Z 9 180-199 30 30 27 53 53 48 F F F <---
33902024-02-03T17:23:58.294Z 10 200-219 28 28 28 51 51 51 F F F
33912024-02-03T17:23:58.294Z 11 220-239 28 28 28 51 51 51 T T F <---
33922024-02-03T17:23:58.294Z 12 240-259 26 26 26 46 46 46 F F F
33932024-02-03T17:23:58.294Z 13 260-279 30 30 28 54 54 51 F F F <---
33942024-02-03T17:23:58.294Z 14 280-299 25 25 25 41 41 41 F F F
33952024-02-03T17:23:58.294Z 15 300-319 14 14 14 23 23 23 F F F
33962024-02-03T17:23:58.295Z 16 320-339 28 28 28 50 50 50 F F F
33972024-02-03T17:23:58.295Z 17 340-359 24 24 24 40 40 40 T T F <---
33982024-02-03T17:23:58.295Z 18 360-379 23 23 23 35 35 35 T T F <---
33992024-02-03T17:23:58.295Z 19 380-399 29 29 29 52 52 52 F F F
34002024-02-03T17:23:58.295Z 20 400-419 30 30 28 54 54 51 F F F <---
34012024-02-03T17:23:58.295Z 21 420-439 30 30 26 54 54 45 F F F <---
34022024-02-03T17:23:58.295Z 22 440-459 29 29 29 52 52 52 T T F <---
34032024-02-03T17:23:58.295Z 23 460-479 28 28 28 49 49 49 T T F <---
34042024-02-03T17:23:58.295Z 24 480-499 26 26 26 42 42 42 F F F
34052024-02-03T17:23:58.295Z 25 500-519 28 28 28 49 49 49 F F F
34062024-02-03T17:23:58.295Z 26 520-539 28 28 28 51 51 51 F F F
34072024-02-03T17:23:58.295Z 27 540-559 23 23 23 35 35 35 F F F
34082024-02-03T17:23:58.295Z 28 560-579 24 24 24 40 40 40 F F F
34092024-02-03T17:23:58.296Z 29 580-599 23 23 23 35 35 35 T T F <---
34102024-02-03T17:23:58.296ZMax gen: 30, Max flush: 54
34112024-02-03T17:23:58.296ZError: Difference in extent metadata found!
34122024-02-03T17:23:58.299ZA Difference in extent metadata is expected here
34132024-02-03T17:23:58.299ZOn loop 15
34142024-02-03T17:23:58.300Z
34152024-02-03T17:23:58.300ZVerifying data now
34162024-02-03T17:24:03.501ZNo change after verify
34172024-02-03T17:24:03.501ZLoop: 15 Downstairs dump after verify (and repair):
34182024-02-03T17:24:03.529ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34192024-02-03T17:24:03.529Z 0 000-019 18 18 18 30 30 30 F F F
34202024-02-03T17:24:03.529Z 1 020-039 31 31 31 55 55 55 F F F
34212024-02-03T17:24:03.529Z 2 040-059 24 24 24 36 36 36 F F F
34222024-02-03T17:24:03.529Z 3 060-079 26 26 26 47 47 47 F F F
34232024-02-03T17:24:03.529Z 4 080-099 31 31 31 55 55 55 F F F
34242024-02-03T17:24:03.529Z 5 100-119 26 26 26 45 45 45 F F F
34252024-02-03T17:24:03.529Z 6 120-139 31 31 31 55 55 55 F F F
34262024-02-03T17:24:03.529Z 7 140-159 31 31 31 55 55 55 F F F
34272024-02-03T17:24:03.536Z 8 160-179 28 28 28 51 51 51 F F F
34282024-02-03T17:24:03.537Z 9 180-199 30 30 30 53 53 53 F F F
34292024-02-03T17:24:03.537Z 10 200-219 28 28 28 51 51 51 F F F
34302024-02-03T17:24:03.537Z 11 220-239 31 31 31 55 55 55 F F F
34312024-02-03T17:24:03.537Z 12 240-259 26 26 26 46 46 46 F F F
34322024-02-03T17:24:03.537Z 13 260-279 30 30 30 54 54 54 F F F
34332024-02-03T17:24:03.537Z 14 280-299 25 25 25 41 41 41 F F F
34342024-02-03T17:24:03.537Z 15 300-319 14 14 14 23 23 23 F F F
34352024-02-03T17:24:03.537Z 16 320-339 28 28 28 50 50 50 F F F
34362024-02-03T17:24:03.537Z 17 340-359 31 31 31 55 55 55 F F F
34372024-02-03T17:24:03.537Z 18 360-379 31 31 31 55 55 55 F F F
34382024-02-03T17:24:03.537Z 19 380-399 29 29 29 52 52 52 F F F
34392024-02-03T17:24:03.537Z 20 400-419 30 30 30 54 54 54 F F F
34402024-02-03T17:24:03.537Z 21 420-439 30 30 30 54 54 54 F F F
34412024-02-03T17:24:03.537Z 22 440-459 31 31 31 55 55 55 F F F
34422024-02-03T17:24:03.537Z 23 460-479 31 31 31 55 55 55 F F F
34432024-02-03T17:24:03.537Z 24 480-499 26 26 26 42 42 42 F F F
34442024-02-03T17:24:03.537Z 25 500-519 28 28 28 49 49 49 F F F
34452024-02-03T17:24:03.537Z 26 520-539 28 28 28 51 51 51 F F F
34462024-02-03T17:24:03.537Z 27 540-559 23 23 23 35 35 35 F F F
34472024-02-03T17:24:03.537Z 28 560-579 24 24 24 40 40 40 F F F
34482024-02-03T17:24:03.537Z 29 580-599 31 31 31 55 55 55 F F F
34492024-02-03T17:24:03.537ZMax gen: 31, Max flush: 55
34502024-02-03T17:24:03.537Z
34512024-02-03T17:24:03.537ZBegin loop 16
34522024-02-03T17:24:03.538ZDownstairs to restart: 0
34532024-02-03T17:24:03.566Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.721929145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257}
34542024-02-03T17:24:03.566Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.722438113Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257}
34552024-02-03T17:24:03.567Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.722483303Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257}
34562024-02-03T17:24:03.567Z{"msg":"Crucible 6c336e22-5303-48f1-9726-f5cfb1332f01 has session id: 496efa53-7afe-4217-a39b-a08f9fd6fbe1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.722519519Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34572024-02-03T17:24:03.578Z{"msg":"Upstairs opts: Upstairs UUID: 6c336e22-5303-48f1-9726-f5cfb1332f01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.722604041Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34582024-02-03T17:24:03.578Z{"msg":"Crucible stats registered with UUID: 6c336e22-5303-48f1-9726-f5cfb1332f01","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.722648425Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34592024-02-03T17:24:03.578ZCrucible runtime is spawned
34602024-02-03T17:24:03.578Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.72286526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257}
34612024-02-03T17:24:03.578Z{"msg":"6c336e22-5303-48f1-9726-f5cfb1332f01 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723001882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34622024-02-03T17:24:03.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723062991Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34632024-02-03T17:24:03.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723089554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34642024-02-03T17:24:03.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723112007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34652024-02-03T17:24:03.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723153904Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34662024-02-03T17:24:03.578Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723177981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34672024-02-03T17:24:03.578Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723200566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34682024-02-03T17:24:03.578Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.72322259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34692024-02-03T17:24:03.578Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723284843Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34702024-02-03T17:24:03.578Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723353659Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34712024-02-03T17:24:03.578Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723635948Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34722024-02-03T17:24:03.578Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.72375443Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34732024-02-03T17:24:03.578Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.723802453Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34742024-02-03T17:24:03.578Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.724801505Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34752024-02-03T17:24:03.578Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.724955423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34762024-02-03T17:24:03.578Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.725058765Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34772024-02-03T17:24:03.579Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.725132139Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34782024-02-03T17:24:03.579Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.725771291Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34792024-02-03T17:24:03.579Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.725990681Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34802024-02-03T17:24:03.579Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.726196098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34812024-02-03T17:24:03.579Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.726300939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34822024-02-03T17:24:03.579Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.726667629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34832024-02-03T17:24:03.579Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.726731408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34842024-02-03T17:24:03.579Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.726777754Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34852024-02-03T17:24:03.579Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.726852049Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34862024-02-03T17:24:03.579Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.726921288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34872024-02-03T17:24:03.579Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.726949294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34882024-02-03T17:24:03.579Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.727285326Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34892024-02-03T17:24:03.579Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.727965693Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34902024-02-03T17:24:03.579Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728035857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34912024-02-03T17:24:03.579Z{"msg":"[0]R flush_numbers[0..12]: [30, 55, 36, 47, 55, 45, 55, 55, 51, 53, 51, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728063206Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34922024-02-03T17:24:03.579Z{"msg":"[0]R generation[0..12]: [18, 31, 24, 26, 31, 26, 31, 31, 28, 30, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728086846Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34932024-02-03T17:24:03.579Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728110937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34942024-02-03T17:24:03.579Z{"msg":"[1]R flush_numbers[0..12]: [30, 55, 36, 47, 55, 45, 55, 55, 51, 53, 51, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728157294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34952024-02-03T17:24:03.579Z{"msg":"[1]R generation[0..12]: [18, 31, 24, 26, 31, 26, 31, 31, 28, 30, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728200933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34962024-02-03T17:24:03.579Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728226502Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34972024-02-03T17:24:03.579Z{"msg":"[2]R flush_numbers[0..12]: [30, 55, 36, 47, 55, 45, 55, 55, 51, 53, 51, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728280786Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34982024-02-03T17:24:03.579Z{"msg":"[2]R generation[0..12]: [18, 31, 24, 26, 31, 26, 31, 31, 28, 30, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728305408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
34992024-02-03T17:24:03.579ZWait for a query_work_queue command to finish before sending IO
35002024-02-03T17:24:03.580Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728332498Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35012024-02-03T17:24:03.580Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728381622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35022024-02-03T17:24:03.580Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728405609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35032024-02-03T17:24:03.580Z{"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.72842694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35042024-02-03T17:24:03.580Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728448816Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35052024-02-03T17:24:03.580Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728489739Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35062024-02-03T17:24:03.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728532254Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35072024-02-03T17:24:03.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728555352Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35082024-02-03T17:24:03.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728576988Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35092024-02-03T17:24:03.580Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.72861879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35102024-02-03T17:24:03.580Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728642078Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35112024-02-03T17:24:03.580Z{"msg":"6c336e22-5303-48f1-9726-f5cfb1332f01 is now active with session: 496efa53-7afe-4217-a39b-a08f9fd6fbe1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.72866394Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35122024-02-03T17:24:03.580Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728708237Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257,"session_id":"496efa53-7afe-4217-a39b-a08f9fd6fbe1"}
35132024-02-03T17:24:03.580Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:02.728730378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257}
35142024-02-03T17:24:03.580ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35152024-02-03T17:24:03.580ZLoading write count information from file "/tmp/test_repair_verify.data"
35162024-02-03T17:24:03.580ZRun Repair workload
35172024-02-03T17:24:03.580Z01/30 Flush
35182024-02-03T17:24:03.580Z02/30 Write block 184 len 12288 data: 4 3 4
35192024-02-03T17:24:03.587Z03/30 Flush
35202024-02-03T17:24:03.587Z04/30 Write block 130 len 20480 data: 4 4 4 4 4
35212024-02-03T17:24:03.587Z05/30 Write block 578 len 16384 data: 5 6 6 5
35222024-02-03T17:24:03.587Z06/30 Read block 4 len 20480
35232024-02-03T17:24:03.587Z07/30 Read block 424 len 40960
35242024-02-03T17:24:03.587Z08/30 Write block 545 len 28672 data: 3 3 3 3 3 3 5
35252024-02-03T17:24:03.587Z09/30 Read block 387 len 20480
35262024-02-03T17:24:03.591Z10/30 Write block 291 len 16384 data: 4 5 5 4
35272024-02-03T17:24:03.591Z11/30 Flush
35282024-02-03T17:24:03.591Z12/30 Write block 36 len 16384 data: 4 4 4 5
35292024-02-03T17:24:03.591Z13/30 Flush
35302024-02-03T17:24:03.594Z14/30 Write block 41 len 20480 data: 4 3 3 4 3
35312024-02-03T17:24:03.594Z15/30 Read block 114 len 16384
35322024-02-03T17:24:03.598Z16/30 Write block 565 len 20480 data: 4 5 5 5 4
35332024-02-03T17:24:03.598Z17/30 Read block 412 len 12288
35342024-02-03T17:24:03.624Z18/30 Read block 45 len 24576
35352024-02-03T17:24:03.624Z19/30 Read block 239 len 16384
35362024-02-03T17:24:03.624Z20/30 Read block 99 len 4096
35372024-02-03T17:24:03.624Z21/30 Read block 378 len 28672
35382024-02-03T17:24:03.624Z22/30 Write block 501 len 24576 data: 5 5 5 4 5 7
35392024-02-03T17:24:03.624Z23/30 Read block 49 len 36864
35402024-02-03T17:24:03.624Z24/30 Flush
35412024-02-03T17:24:03.624Z25/30 Write block 254 len 8192 data: 4 5
35422024-02-03T17:24:03.624Z26/30 Write block 337 len 4096 data: 3
35432024-02-03T17:24:03.624Z27/30 Read block 127 len 32768
35442024-02-03T17:24:03.624Z28/30 Read block 224 len 24576
35452024-02-03T17:24:03.624Z29/30 Read block 484 len 20480
35462024-02-03T17:24:03.624Z30/30 Read block 338 len 32768
35472024-02-03T17:24:03.624Z----------------------------------------------------------------
35482024-02-03T17:24:03.624Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35492024-02-03T17:24:03.624ZStates: Active Active Active
35502024-02-03T17:24:03.624ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35512024-02-03T17:24:03.624Z 1 Acked 1000 Flush 0 Sent Done Done false
35522024-02-03T17:24:03.625Z 2 Acked 1001 Write 3 Sent Done Done false
35532024-02-03T17:24:03.625Z 3 Acked 1002 Flush 0 Sent Done Done false
35542024-02-03T17:24:03.625Z 4 Acked 1003 Write 5 Sent Done Done false
35552024-02-03T17:24:03.625Z 5 Acked 1004 Write 4 Sent Done Done false
35562024-02-03T17:24:03.625Z 6 Acked 1005 Read 5 Sent Done Done false
35572024-02-03T17:24:03.625Z 7 Acked 1006 Read 10 Sent Done Done false
35582024-02-03T17:24:03.625Z 8 Acked 1007 Write 7 Sent Done Done false
35592024-02-03T17:24:03.625Z 9 Acked 1008 Read 5 Sent Done Done false
35602024-02-03T17:24:03.625Z 10 Acked 1009 Write 4 Sent Done Done false
35612024-02-03T17:24:03.625Z 11 Acked 1010 Flush 0 Sent Done Done false
35622024-02-03T17:24:03.625Z 12 Acked 1011 Write 4 Sent Done Done false
35632024-02-03T17:24:03.625Z 13 Acked 1012 Flush 0 Sent Done Done false
35642024-02-03T17:24:03.625Z 14 Acked 1013 Write 5 Sent Done Done false
35652024-02-03T17:24:03.625Z 15 Acked 1014 Read 4 Sent Done Done false
35662024-02-03T17:24:03.625Z 16 Acked 1015 Write 5 Sent Done Done false
35672024-02-03T17:24:03.625Z 17 Acked 1016 Read 3 Sent Done Done false
35682024-02-03T17:24:03.625Z 18 Acked 1017 Read 6 Sent Done Done false
35692024-02-03T17:24:03.625Z 19 Acked 1018 Read 4 Sent Done Done false
35702024-02-03T17:24:03.625Z 20 Acked 1019 Read 1 Sent Done Done false
35712024-02-03T17:24:03.625Z 21 Acked 1020 Read 7 Sent Done Done false
35722024-02-03T17:24:03.625Z 22 Acked 1021 Write 6 Sent Done Done false
35732024-02-03T17:24:03.625Z 23 Acked 1022 Read 9 Sent Done Done false
35742024-02-03T17:24:03.625Z 24 Acked 1023 Flush 0 Sent Done Done false
35752024-02-03T17:24:03.625Z 25 Acked 1024 Write 2 Sent Done Done false
35762024-02-03T17:24:03.625Z 26 Acked 1025 Write 1 Sent Done Done false
35772024-02-03T17:24:03.626Z 27 Acked 1026 Read 8 Sent Done Done false
35782024-02-03T17:24:03.626Z 28 Acked 1027 Read 6 Sent Done Done false
35792024-02-03T17:24:03.626Z 29 Acked 1028 Read 5 Sent Done Done false
35802024-02-03T17:24:03.626Z 30 Acked 1029 Read 8 Sent Done Done false
35812024-02-03T17:24:03.626Z STATES DS:0 DS:1 DS:2 TOTAL
35822024-02-03T17:24:03.626Z New 0 0 0 0
35832024-02-03T17:24:03.626Z Sent 30 0 0 30
35842024-02-03T17:24:03.626Z Done 0 30 30 60
35852024-02-03T17:24:03.626Z Skipped 0 0 0 0
35862024-02-03T17:24:03.626Z Error 0 0 0 0
35872024-02-03T17:24:03.626ZLast Flush: 0 1023 1023
35882024-02-03T17:24:03.626ZDownstairs last five completed:
35892024-02-03T17:24:03.626ZUpstairs last five completed: 30 29 28 27 26
35902024-02-03T17:24:03.626Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:02.778239175Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1257}
35912024-02-03T17:24:03.626ZWrote out file "/tmp/test_repair_verify.data"
35922024-02-03T17:24:03.631Z
35932024-02-03T17:24:03.636ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35942024-02-03T17:24:03.659ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35952024-02-03T17:24:03.659Z 0 000-019 18 18 18 30 30 30 F F F
35962024-02-03T17:24:03.663Z 1 020-039 31 32 32 55 59 59 F F F <---
35972024-02-03T17:24:03.664Z 2 040-059 24 32 32 36 60 60 F F F <---
35982024-02-03T17:24:03.664Z 3 060-079 26 26 26 47 47 47 F F F
35992024-02-03T17:24:03.664Z 4 080-099 31 31 31 55 55 55 F F F
36002024-02-03T17:24:03.664Z 5 100-119 26 26 26 45 45 45 F F F
36012024-02-03T17:24:03.664Z 6 120-139 31 32 32 55 58 58 F F F <---
36022024-02-03T17:24:03.664Z 7 140-159 31 31 31 55 55 55 F F F
36032024-02-03T17:24:03.664Z 8 160-179 28 28 28 51 51 51 F F F
36042024-02-03T17:24:03.664Z 9 180-199 30 32 32 53 57 57 F F F <---
36052024-02-03T17:24:03.664Z 10 200-219 28 28 28 51 51 51 F F F
36062024-02-03T17:24:03.664Z 11 220-239 31 31 31 55 55 55 F F F
36072024-02-03T17:24:03.664Z 12 240-259 26 26 26 46 46 46 F T T <---
36082024-02-03T17:24:03.664Z 13 260-279 30 30 30 54 54 54 F F F
36092024-02-03T17:24:03.664Z 14 280-299 25 32 32 41 58 58 F F F <---
36102024-02-03T17:24:03.664Z 15 300-319 14 14 14 23 23 23 F F F
36112024-02-03T17:24:03.664Z 16 320-339 28 28 28 50 50 50 F T T <---
36122024-02-03T17:24:03.664Z 17 340-359 31 31 31 55 55 55 F F F
36132024-02-03T17:24:03.664Z 18 360-379 31 31 31 55 55 55 F F F
36142024-02-03T17:24:03.664Z 19 380-399 29 29 29 52 52 52 F F F
36152024-02-03T17:24:03.664Z 20 400-419 30 30 30 54 54 54 F F F
36162024-02-03T17:24:03.664Z 21 420-439 30 30 30 54 54 54 F F F
36172024-02-03T17:24:03.664Z 22 440-459 31 31 31 55 55 55 F F F
36182024-02-03T17:24:03.664Z 23 460-479 31 31 31 55 55 55 F F F
36192024-02-03T17:24:03.664Z 24 480-499 26 26 26 42 42 42 F F F
36202024-02-03T17:24:03.665Z 25 500-519 28 32 32 49 60 60 F F F <---
36212024-02-03T17:24:03.665Z 26 520-539 28 28 28 51 51 51 F F F
36222024-02-03T17:24:03.665Z 27 540-559 23 32 32 35 58 58 F F F <---
36232024-02-03T17:24:03.665Z 28 560-579 24 32 32 40 60 60 F F F <---
36242024-02-03T17:24:03.665Z 29 580-599 31 32 32 55 58 58 F F F <---
36252024-02-03T17:24:03.665ZMax gen: 32, Max flush: 60
36262024-02-03T17:24:03.665ZError: Difference in extent metadata found!
36272024-02-03T17:24:03.669ZA Difference in extent metadata is expected here
36282024-02-03T17:24:03.669ZOn loop 16
36292024-02-03T17:24:03.669Z
36302024-02-03T17:24:03.669ZVerifying data now
36312024-02-03T17:24:08.658ZNo change after verify
36322024-02-03T17:24:08.658ZLoop: 16 Downstairs dump after verify (and repair):
36332024-02-03T17:24:08.682ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36342024-02-03T17:24:08.683Z 0 000-019 18 18 18 30 30 30 F F F
36352024-02-03T17:24:08.686Z 1 020-039 32 32 32 59 59 59 F F F
36362024-02-03T17:24:08.686Z 2 040-059 32 32 32 60 60 60 F F F
36372024-02-03T17:24:08.686Z 3 060-079 26 26 26 47 47 47 F F F
36382024-02-03T17:24:08.687Z 4 080-099 31 31 31 55 55 55 F F F
36392024-02-03T17:24:08.687Z 5 100-119 26 26 26 45 45 45 F F F
36402024-02-03T17:24:08.687Z 6 120-139 32 32 32 58 58 58 F F F
36412024-02-03T17:24:08.687Z 7 140-159 31 31 31 55 55 55 F F F
36422024-02-03T17:24:08.687Z 8 160-179 28 28 28 51 51 51 F F F
36432024-02-03T17:24:08.687Z 9 180-199 32 32 32 57 57 57 F F F
36442024-02-03T17:24:08.687Z 10 200-219 28 28 28 51 51 51 F F F
36452024-02-03T17:24:08.687Z 11 220-239 31 31 31 55 55 55 F F F
36462024-02-03T17:24:08.687Z 12 240-259 33 33 33 61 61 61 F F F
36472024-02-03T17:24:08.687Z 13 260-279 30 30 30 54 54 54 F F F
36482024-02-03T17:24:08.688Z 14 280-299 32 32 32 58 58 58 F F F
36492024-02-03T17:24:08.688Z 15 300-319 14 14 14 23 23 23 F F F
36502024-02-03T17:24:08.688Z 16 320-339 33 33 33 61 61 61 F F F
36512024-02-03T17:24:08.688Z 17 340-359 31 31 31 55 55 55 F F F
36522024-02-03T17:24:08.688Z 18 360-379 31 31 31 55 55 55 F F F
36532024-02-03T17:24:08.688Z 19 380-399 29 29 29 52 52 52 F F F
36542024-02-03T17:24:08.688Z 20 400-419 30 30 30 54 54 54 F F F
36552024-02-03T17:24:08.688Z 21 420-439 30 30 30 54 54 54 F F F
36562024-02-03T17:24:08.688Z 22 440-459 31 31 31 55 55 55 F F F
36572024-02-03T17:24:08.688Z 23 460-479 31 31 31 55 55 55 F F F
36582024-02-03T17:24:08.688Z 24 480-499 26 26 26 42 42 42 F F F
36592024-02-03T17:24:08.689Z 25 500-519 32 32 32 60 60 60 F F F
36602024-02-03T17:24:08.689Z 26 520-539 28 28 28 51 51 51 F F F
36612024-02-03T17:24:08.689Z 27 540-559 32 32 32 58 58 58 F F F
36622024-02-03T17:24:08.689Z 28 560-579 32 32 32 60 60 60 F F F
36632024-02-03T17:24:08.689Z 29 580-599 32 32 32 58 58 58 F F F
36642024-02-03T17:24:08.689ZMax gen: 33, Max flush: 61
36652024-02-03T17:24:08.694Z
36662024-02-03T17:24:08.694ZBegin loop 17
36672024-02-03T17:24:08.694ZDownstairs to restart: 0
36682024-02-03T17:24:08.720Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.87903299Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265}
36692024-02-03T17:24:08.720Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.879609355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265}
36702024-02-03T17:24:08.739Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.879660126Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265}
36712024-02-03T17:24:08.739Z{"msg":"Crucible runtime is spawned
36722024-02-03T17:24:08.740ZCrucible a7f0ba3c-1253-41b6-897c-34af8d34865c has session id: 194cd5bf-a408-4325-907e-e0aa4204e935","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.879863148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36732024-02-03T17:24:08.740Z{"msg":"Upstairs opts: Upstairs UUID: a7f0ba3c-1253-41b6-897c-34af8d34865c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.87997193Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36742024-02-03T17:24:08.740Z{"msg":"Crucible stats registered with UUID: a7f0ba3c-1253-41b6-897c-34af8d34865c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880071389Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36752024-02-03T17:24:08.740Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88014353Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265}
36762024-02-03T17:24:08.740Z{"msg":"a7f0ba3c-1253-41b6-897c-34af8d34865c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880214152Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36772024-02-03T17:24:08.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880251309Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36782024-02-03T17:24:08.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880318914Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36792024-02-03T17:24:08.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880361309Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36802024-02-03T17:24:08.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880437635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36812024-02-03T17:24:08.740Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880478282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36822024-02-03T17:24:08.740Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880542775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36832024-02-03T17:24:08.740Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88058444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36842024-02-03T17:24:08.741Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88062462Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36852024-02-03T17:24:08.741Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880662936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36862024-02-03T17:24:08.741Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880702853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36872024-02-03T17:24:08.741Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88079093Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36882024-02-03T17:24:08.741Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.880836156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36892024-02-03T17:24:08.741Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.882134479Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36902024-02-03T17:24:08.741Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.882202596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36912024-02-03T17:24:08.741Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.882355143Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36922024-02-03T17:24:08.741Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.882439875Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36932024-02-03T17:24:08.741Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.883159893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36942024-02-03T17:24:08.741Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.883219145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36952024-02-03T17:24:08.741Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88324639Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36962024-02-03T17:24:08.741Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.883352367Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36972024-02-03T17:24:08.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.884097053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36982024-02-03T17:24:08.742Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.884159523Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
36992024-02-03T17:24:08.742Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.884186284Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37002024-02-03T17:24:08.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.884207256Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37012024-02-03T17:24:08.742Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88425334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37022024-02-03T17:24:08.742Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.884277819Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37032024-02-03T17:24:08.742Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.884507961Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37042024-02-03T17:24:08.742Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885198713Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37052024-02-03T17:24:08.742Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88531117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37062024-02-03T17:24:08.742Z{"msg":"[0]R flush_numbers[0..12]: [30, 59, 60, 47, 55, 45, 58, 55, 51, 57, 51, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885377226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37072024-02-03T17:24:08.742Z{"msg":"[0]R generation[0..12]: [18, 32, 32, 26, 31, 26, 32, 31, 28, 32, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885406052Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37082024-02-03T17:24:08.742Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885451054Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37092024-02-03T17:24:08.742Z{"msg":"[1]R flush_numbers[0..12]: [30, 59, 60, 47, 55, 45, 58, 55, 51, 57, 51, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88547667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37102024-02-03T17:24:08.743Z{"msg":"[1]R generation[0..12]: [18, 32, 32, 26, 31, 26, 32, 31, 28, 32, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885526112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37112024-02-03T17:24:08.743Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
37122024-02-03T17:24:08.743Z2024-02-03T17:24:07.885551798Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37132024-02-03T17:24:08.743Z{"msg":"[2]R flush_numbers[0..12]: [30, 59, 60, 47, 55, 45, 58, 55, 51, 57, 51, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885599123Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37142024-02-03T17:24:08.743Z{"msg":"[2]R generation[0..12]: [18, 32, 32, 26, 31, 26, 32, 31, 28, 32, 28, 31]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885624082Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37152024-02-03T17:24:08.748Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885672541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37162024-02-03T17:24:08.748Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885788865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37172024-02-03T17:24:08.748Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.88585303Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37182024-02-03T17:24:08.748Z{"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885922745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37192024-02-03T17:24:08.748Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.885952606Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37202024-02-03T17:24:08.748Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886014221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37212024-02-03T17:24:08.748Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886087678Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37222024-02-03T17:24:08.748Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886115391Z"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37232024-02-03T17:24:08.748Z,"hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37242024-02-03T17:24:08.748Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886223714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37252024-02-03T17:24:08.748Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886275349Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37262024-02-03T17:24:08.748Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886298714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37272024-02-03T17:24:08.749Z{"msg":"a7f0ba3c-1253-41b6-897c-34af8d34865c is now active with session: 194cd5bf-a408-4325-907e-e0aa4204e935","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886339207Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37282024-02-03T17:24:08.749Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886360993Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265,"session_id":"194cd5bf-a408-4325-907e-e0aa4204e935"}
37292024-02-03T17:24:08.749Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:07.886400453Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265}
37302024-02-03T17:24:08.749ZLoading write count information from file "/tmp/test_repair_verify.data"
37312024-02-03T17:24:08.749ZRun Repair workload
37322024-02-03T17:24:08.749Z01/30 Write block 297 len 8192 data: 3 2
37332024-02-03T17:24:08.749Z02/30 Read block 453 len 16384
37342024-02-03T17:24:08.749Z03/30 Write block 313 len 16384 data: 2 2 2 2
37352024-02-03T17:24:08.749Z04/30 Write block 395 len 12288 data: 7 6 5
37362024-02-03T17:24:08.749Z05/30 Read block 259 len 4096
37372024-02-03T17:24:08.749Z06/30 Read block 211 len 24576
37382024-02-03T17:24:08.749Z07/30 Read block 55 len 32768
37392024-02-03T17:24:08.750Z08/30 Read block 342 len 32768
37402024-02-03T17:24:08.750Z09/30 Read block 114 len 28672
37412024-02-03T17:24:08.750Z10/30 Write block 416 len 20480 data: 3 4 3 3 3
37422024-02-03T17:24:08.750Z11/30 Write block 398 len 40960 data: 6 6 5 4 5 5 4 5 5 5
37432024-02-03T17:24:08.750Z12/30 Write block 57 len 40960 data: 3 3 3 2 2 2 3 3 2 2
37442024-02-03T17:24:08.750Z13/30 Write block 356 len 4096 data: 3
37452024-02-03T17:24:08.753Z14/30 Read block 122 len 32768
37462024-02-03T17:24:08.754Z15/30 Read block 364 len 32768
37472024-02-03T17:24:08.754Z16/30 Write block 7 len 40960 data: 4 5 4 2 3 3 3 3 2 2
37482024-02-03T17:24:08.754Z17/30 Write block 50 len 32768 data: 5 5 5 4 2 3 4 4
37492024-02-03T17:24:08.754Z18/30 Read block 411 len 20480
37502024-02-03T17:24:08.758Z19/30 Read block 234 len 32768
37512024-02-03T17:24:08.758Z20/30 Read block 455 len 40960
37522024-02-03T17:24:08.778Z21/30 Write block 164 len 8192 data: 4 4
37532024-02-03T17:24:08.778Z22/30 Read block 113 len 40960
37542024-02-03T17:24:08.778Z23/30 Read block 43 len 24576
37552024-02-03T17:24:08.778Z24/30 Write block 247 len 16384 data: 2 2 2 3
37562024-02-03T17:24:08.778Z25/30 Write block 285 len 40960 data: 3 3 4 4 4 4 5 6 6 5
37572024-02-03T17:24:08.778Z26/30 Read block 56 len 28672
37582024-02-03T17:24:08.778Z27/30 Read block 138 len 12288
37592024-02-03T17:24:08.778Z28/30 Read block 542 len 16384
37602024-02-03T17:24:08.779Z29/30 Read block 312 len 16384
37612024-02-03T17:24:08.779Z30/30 Write block 215 len 36864 data: 6 4 5 5 5 4 7 7 4
37622024-02-03T17:24:08.779Z----------------------------------------------------------------
37632024-02-03T17:24:08.779Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37642024-02-03T17:24:08.779ZStates: Active Active Active
37652024-02-03T17:24:08.779ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37662024-02-03T17:24:08.779Z 1 Acked 1000 Write 2 Done Done Done false
37672024-02-03T17:24:08.780Z 2 Acked 1001 Read 4 Done Done Done false
37682024-02-03T17:24:08.780Z 3 Acked 1002 Write 4 Done Done Done false
37692024-02-03T17:24:08.780Z 4 Acked 1003 Write 3 Done Done Done false
37702024-02-03T17:24:08.780Z 5 Acked 1004 Read 1 Done Done Done false
37712024-02-03T17:24:08.780Z 6 Acked 1005 Read 6 Sent Done Done false
37722024-02-03T17:24:08.780Z 7 Acked 1006 Read 8 Sent Done Done false
37732024-02-03T17:24:08.780Z 8 Acked 1007 Read 8 Sent Done Done false
37742024-02-03T17:24:08.780Z 9 Acked 1008 Read 7 Sent Done Done false
37752024-02-03T17:24:08.780Z 10 Acked 1009 Write 5 Sent Done Done false
37762024-02-03T17:24:08.781Z 11 Acked 1010 Write 10 Sent Done Done false
37772024-02-03T17:24:08.781Z 12 Acked 1011 Write 10 Sent Done Done false
37782024-02-03T17:24:08.781Z 13 Acked 1012 Write 1 Sent Done Done false
37792024-02-03T17:24:08.781Z 14 Acked 1013 Read 8 Sent Done Done false
37802024-02-03T17:24:08.781Z 15 Acked 1014 Read 8 Sent Done Done false
37812024-02-03T17:24:08.781Z 16 Acked 1015 Write 10 Sent Done Done false
37822024-02-03T17:24:08.781Z 17 Acked 1016 Write 8 Sent Done Done false
37832024-02-03T17:24:08.781Z 18 Acked 1017 Read 5 Sent Done Done false
37842024-02-03T17:24:08.781Z 19 Acked 1018 Read 8 Sent Done Done false
37852024-02-03T17:24:08.781Z 20 Acked 1019 Read 10 Sent Done Done false
37862024-02-03T17:24:08.781Z 21 Acked 1020 Write 2 Sent Done Done false
37872024-02-03T17:24:08.781Z 22 Acked 1021 Read 10 Sent Done Done false
37882024-02-03T17:24:08.781Z 23 Acked 1022 Read 6 Sent Done Done false
37892024-02-03T17:24:08.781Z 24 Acked 1023 Write 4 Sent Done Done false
37902024-02-03T17:24:08.781Z 25 Acked 1024 Write 10 Sent Done Done false
37912024-02-03T17:24:08.781Z 26 Acked 1025 Read 7 Sent Done Done false
37922024-02-03T17:24:08.782Z 27 Acked 1026 Read 3 Sent Done Done false
37932024-02-03T17:24:08.782Z 28 Acked 1027 Read 4 Sent Done Done false
37942024-02-03T17:24:08.782Z 29 Acked 1028 Read 4 Sent Done Done false
37952024-02-03T17:24:08.782Z 30 Acked 1029 Write 9 Sent Sent Sent false
37962024-02-03T17:24:08.782Z STATES DS:0 DS:1 DS:2 TOTAL
37972024-02-03T17:24:08.782Z New 0 0 0 0
37982024-02-03T17:24:08.782Z Sent 25 1 1 27
37992024-02-03T17:24:08.782Z Done 5 29 29 63
38002024-02-03T17:24:08.782Z Skipped 0 0 0 0
38012024-02-03T17:24:08.782Z Error 0 0 0 0
38022024-02-03T17:24:08.782ZLast Flush: 0 0 0
38032024-02-03T17:24:08.782ZDownstairs last five completed:
38042024-02-03T17:24:08.782ZUpstairs last five completed: 30 29 28 27 26
38052024-02-03T17:24:08.782Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:07.932883939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1265}
38062024-02-03T17:24:08.782ZWrote out file "/tmp/test_repair_verify.data"
38072024-02-03T17:24:08.788Z
38082024-02-03T17:24:08.792ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38092024-02-03T17:24:08.822ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38102024-02-03T17:24:08.822Z 0 000-019 18 18 18 30 30 30 F T T <---
38112024-02-03T17:24:08.822Z 1 020-039 32 32 32 59 59 59 F F F
38122024-02-03T17:24:08.822Z 2 040-059 32 32 32 60 60 60 F T T <---
38132024-02-03T17:24:08.822Z 3 060-079 26 26 26 47 47 47 F T T <---
38142024-02-03T17:24:08.823Z 4 080-099 31 31 31 55 55 55 F F F
38152024-02-03T17:24:08.823Z 5 100-119 26 26 26 45 45 45 F F F
38162024-02-03T17:24:08.826Z 6 120-139 32 32 32 58 58 58 F F F
38172024-02-03T17:24:08.826Z 7 140-159 31 31 31 55 55 55 F F F
38182024-02-03T17:24:08.826Z 8 160-179 28 28 28 51 51 51 F T T <---
38192024-02-03T17:24:08.826Z 9 180-199 32 32 32 57 57 57 F F F
38202024-02-03T17:24:08.827Z 10 200-219 28 28 28 51 51 51 F T T <---
38212024-02-03T17:24:08.827Z 11 220-239 31 31 31 55 55 55 F T T <---
38222024-02-03T17:24:08.827Z 12 240-259 33 33 33 61 61 61 F T T <---
38232024-02-03T17:24:08.827Z 13 260-279 30 30 30 54 54 54 F F F
38242024-02-03T17:24:08.827Z 14 280-299 32 32 32 58 58 58 T T T
38252024-02-03T17:24:08.827Z 15 300-319 14 14 14 23 23 23 T T T
38262024-02-03T17:24:08.827Z 16 320-339 33 33 33 61 61 61 F F F
38272024-02-03T17:24:08.827Z 17 340-359 31 31 31 55 55 55 F T T <---
38282024-02-03T17:24:08.827Z 18 360-379 31 31 31 55 55 55 F F F
38292024-02-03T17:24:08.827Z 19 380-399 29 29 29 52 52 52 T T T
38302024-02-03T17:24:08.827Z 20 400-419 30 30 30 54 54 54 F T T <---
38312024-02-03T17:24:08.827Z 21 420-439 30 30 30 54 54 54 F T T <---
38322024-02-03T17:24:08.827Z 22 440-459 31 31 31 55 55 55 F F F
38332024-02-03T17:24:08.827Z 23 460-479 31 31 31 55 55 55 F F F
38342024-02-03T17:24:08.827Z 24 480-499 26 26 26 42 42 42 F F F
38352024-02-03T17:24:08.827Z 25 500-519 32 32 32 60 60 60 F F F
38362024-02-03T17:24:08.827Z 26 520-539 28 28 28 51 51 51 F F F
38372024-02-03T17:24:08.827Z 27 540-559 32 32 32 58 58 58 F F F
38382024-02-03T17:24:08.827Z 28 560-579 32 32 32 60 60 60 F F F
38392024-02-03T17:24:08.827Z 29 580-599 32 32 32 58 58 58 F F F
38402024-02-03T17:24:08.827ZMax gen: 33, Max flush: 61
38412024-02-03T17:24:08.827ZError: Difference in extent metadata found!
38422024-02-03T17:24:08.831ZA Difference in extent metadata is expected here
38432024-02-03T17:24:08.831ZOn loop 17
38442024-02-03T17:24:08.831Z
38452024-02-03T17:24:08.831ZVerifying data now
38462024-02-03T17:24:15.158ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
38472024-02-03T17:24:15.162Z288,289d287
38482024-02-03T17:24:15.162Z< 2,
38492024-02-03T17:24:15.162Z< 2,
38502024-02-03T17:24:15.162Z292,293d289
38512024-02-03T17:24:15.162Z< 3,
38522024-02-03T17:24:15.162Z< 3,
38532024-02-03T17:24:15.162Z294a291,293
38542024-02-03T17:24:15.162Z> 4,
38552024-02-03T17:24:15.162Z> 4,
38562024-02-03T17:24:15.162Z> 4,
38572024-02-03T17:24:15.162Z295a295,296
38582024-02-03T17:24:15.162Z> 6,
38592024-02-03T17:24:15.162Z> 6,
38602024-02-03T17:24:15.162Z297d297
38612024-02-03T17:24:15.162Z< 4,
38622024-02-03T17:24:15.162Z400a401,402
38632024-02-03T17:24:15.162Z> 6,
38642024-02-03T17:24:15.162Z> 6,
38652024-02-03T17:24:15.162Z402,403d403
38662024-02-03T17:24:15.162Z< 5,
38672024-02-03T17:24:15.162Z< 5,
38682024-02-03T17:24:15.162Zdiff found after verify
38692024-02-03T17:24:15.166ZLoop: 17 Downstairs dump after verify (and repair):
38702024-02-03T17:24:15.189ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38712024-02-03T17:24:15.189Z 0 000-019 34 34 34 62 62 62 F F F
38722024-02-03T17:24:15.192Z 1 020-039 32 32 32 59 59 59 F F F
38732024-02-03T17:24:15.192Z 2 040-059 34 34 34 62 62 62 F F F
38742024-02-03T17:24:15.193Z 3 060-079 34 34 34 62 62 62 F F F
38752024-02-03T17:24:15.193Z 4 080-099 31 31 31 55 55 55 F F F
38762024-02-03T17:24:15.193Z 5 100-119 26 26 26 45 45 45 F F F
38772024-02-03T17:24:15.193Z 6 120-139 32 32 32 58 58 58 F F F
38782024-02-03T17:24:15.193Z 7 140-159 31 31 31 55 55 55 F F F
38792024-02-03T17:24:15.193Z 8 160-179 34 34 34 62 62 62 F F F
38802024-02-03T17:24:15.193Z 9 180-199 32 32 32 57 57 57 F F F
38812024-02-03T17:24:15.193Z 10 200-219 34 34 34 62 62 62 F F F
38822024-02-03T17:24:15.193Z 11 220-239 34 34 34 62 62 62 F F F
38832024-02-03T17:24:15.193Z 12 240-259 34 34 34 62 62 62 F F F
38842024-02-03T17:24:15.193Z 13 260-279 30 30 30 54 54 54 F F F
38852024-02-03T17:24:15.193Z 14 280-299 34 34 34 62 62 62 F F F
38862024-02-03T17:24:15.193Z 15 300-319 34 34 34 62 62 62 F F F
38872024-02-03T17:24:15.193Z 16 320-339 33 33 33 61 61 61 F F F
38882024-02-03T17:24:15.193Z 17 340-359 34 34 34 62 62 62 F F F
38892024-02-03T17:24:15.193Z 18 360-379 31 31 31 55 55 55 F F F
38902024-02-03T17:24:15.193Z 19 380-399 34 34 34 62 62 62 F F F
38912024-02-03T17:24:15.193Z 20 400-419 34 34 34 62 62 62 F F F
38922024-02-03T17:24:15.193Z 21 420-439 34 34 34 62 62 62 F F F
38932024-02-03T17:24:15.193Z 22 440-459 31 31 31 55 55 55 F F F
38942024-02-03T17:24:15.193Z 23 460-479 31 31 31 55 55 55 F F F
38952024-02-03T17:24:15.193Z 24 480-499 26 26 26 42 42 42 F F F
38962024-02-03T17:24:15.193Z 25 500-519 32 32 32 60 60 60 F F F
38972024-02-03T17:24:15.193Z 26 520-539 28 28 28 51 51 51 F F F
38982024-02-03T17:24:15.194Z 27 540-559 32 32 32 58 58 58 F F F
38992024-02-03T17:24:15.194Z 28 560-579 32 32 32 60 60 60 F F F
39002024-02-03T17:24:15.194Z 29 580-599 32 32 32 58 58 58 F F F
39012024-02-03T17:24:15.194ZMax gen: 34, Max flush: 62
39022024-02-03T17:24:15.197Z
39032024-02-03T17:24:15.197ZBegin loop 18
39042024-02-03T17:24:15.197ZDownstairs to restart: 0
39052024-02-03T17:24:15.229Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.38808044Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276}
39062024-02-03T17:24:15.229Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.388749583Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276}
39072024-02-03T17:24:15.233Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
39082024-02-03T17:24:15.233Z,"time":"2024-02-03T17:24:14.388833742Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276}
39092024-02-03T17:24:15.233Z{"msg":"Crucible 34f1b8f4-04f1-4ed9-881c-ef043a18d2c4 has session id: c06e11bc-d1ad-4445-a067-99fe0346548f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389023825Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39102024-02-03T17:24:15.233Z{"msg":"Upstairs opts: Upstairs UUID: 34f1b8f4-04f1-4ed9-881c-ef043a18d2c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389128653Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39112024-02-03T17:24:15.233Z{"msg":"Crucible stats registered with UUID: 34f1b8f4-04f1-4ed9-881c-ef043a18d2c4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389158972Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39122024-02-03T17:24:15.233Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389217131Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276}
39132024-02-03T17:24:15.233Z{"msg":"34f1b8f4-04f1-4ed9-881c-ef043a18d2c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389241823Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39142024-02-03T17:24:15.233Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.38930442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39152024-02-03T17:24:15.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389337754Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39162024-02-03T17:24:15.233Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389393433Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39172024-02-03T17:24:15.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389420479Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39182024-02-03T17:24:15.233Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.38949261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39192024-02-03T17:24:15.233Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389674745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39202024-02-03T17:24:15.233Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389730264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39212024-02-03T17:24:15.233Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389774911Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39222024-02-03T17:24:15.233Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389884662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39232024-02-03T17:24:15.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.389982267Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"io task","client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39242024-02-03T17:24:15.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.390032096Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"io task","client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39252024-02-03T17:24:15.233Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.390077386Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"io task","client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39262024-02-03T17:24:15.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.391527218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39272024-02-03T17:24:15.234Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.391626289Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39282024-02-03T17:24:15.234Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.391679013Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39292024-02-03T17:24:15.234Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.391758236Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39302024-02-03T17:24:15.238Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.392012646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39312024-02-03T17:24:15.238Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.392121174Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39322024-02-03T17:24:15.238Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.392530023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39332024-02-03T17:24:15.238Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.393197804Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39342024-02-03T17:24:15.238Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.393284044Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39352024-02-03T17:24:15.238Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.393603745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39362024-02-03T17:24:15.238Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.39366401Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39372024-02-03T17:24:15.238Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.393690367Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39382024-02-03T17:24:15.238Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394294803Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39392024-02-03T17:24:15.238Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394356054Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39402024-02-03T17:24:15.238Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394382363Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39412024-02-03T17:24:15.238Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394403086Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39422024-02-03T17:24:15.238Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394444737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39432024-02-03T17:24:15.238Z{"msg":"[0]R flush_numbers[0..12]: [62, 59, 62, 62, 55, 45, 58, 55, 62, 57, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394469734Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39442024-02-03T17:24:15.238Z{"msg":"[0]R generation[0..12]: [34, 32, 34, 34, 31, 26, 32, 31, 34, 32, 34, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394511719Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39452024-02-03T17:24:15.238Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394538001Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39462024-02-03T17:24:15.238Z{"msg":"[1]R flush_numbers[0..12]: [62, 59, 62, 62, 55, 45, 58, 55, 62, 57, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394581156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39472024-02-03T17:24:15.238Z{"msg":"[1]R generation[0..12]: [34, 32, 34, 34, 31, 26, 32, 31, 34, 32, 34, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394606112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39482024-02-03T17:24:15.239Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.39463918Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39492024-02-03T17:24:15.239Z{"msg":"[2]R flush_numbers[0..12]: [62, 59, 62, 62, 55, 45, 58, 55, 62, 57, 62, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394706235Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39502024-02-03T17:24:15.239Z{"msg":"[2]R generation[0..12]: [34, 32, 34, 34, 31, 26, 32, 31, 34, 32, 34, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394751046Z","hostname":"Wait for a query_work_queue command to finish before sending IO
39512024-02-03T17:24:15.239Zip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39522024-02-03T17:24:15.239Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394853294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39532024-02-03T17:24:15.239Z{"msg":"Max found gen is 35","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394913324Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39542024-02-03T17:24:15.239Z{"msg":"Generation requested: 36 >= found:35","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.394958139Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39552024-02-03T17:24:15.239Z{"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.39501253Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39562024-02-03T17:24:15.239Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395054939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"":"downstairs","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39572024-02-03T17:24:15.239Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395118117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39582024-02-03T17:24:15.239Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395158344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"0","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39592024-02-03T17:24:15.239Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395215997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"1","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39602024-02-03T17:24:15.239Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395259571Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"client":"2","":"downstairs","session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39612024-02-03T17:24:15.239Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395314608Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39622024-02-03T17:24:15.239Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395354899Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39632024-02-03T17:24:15.239Z{"msg":"34f1b8f4-04f1-4ed9-881c-ef043a18d2c4 is now active with session: c06e11bc-d1ad-4445-a067-99fe0346548f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395394642Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39642024-02-03T17:24:15.239Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395431709Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276,"session_id":"c06e11bc-d1ad-4445-a067-99fe0346548f"}
39652024-02-03T17:24:15.239Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:14.395469735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276}
39662024-02-03T17:24:15.239ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39672024-02-03T17:24:15.239ZLoading write count information from file "/tmp/test_repair_verify.data"
39682024-02-03T17:24:15.239ZRun Repair workload
39692024-02-03T17:24:15.239Z01/30 Write block 84 len 4096 data: 4
39702024-02-03T17:24:15.243Z02/30 Write block 522 len 24576 data: 2 2 2 4 5 6
39712024-02-03T17:24:15.243Z03/30 Write block 543 len 36864 data: 2 3 4 4 4 4 4 4 6
39722024-02-03T17:24:15.243Z04/30 Write block 306 len 20480 data: 3 2 2 2 2
39732024-02-03T17:24:15.246Z05/30 Flush
39742024-02-03T17:24:15.249Z06/30 Write block 125 len 28672 data: 5 5 4 4 4 5 5
39752024-02-03T17:24:15.249Z07/30 Read block 335 len 16384
39762024-02-03T17:24:15.252Z08/30 Write block 487 len 4096 data: 2
39772024-02-03T17:24:15.252Z09/30 Write block 150 len 24576 data: 5 5 4 5 6 5
39782024-02-03T17:24:15.255Z10/30 Write block 487 len 32768 data: 3 3 2 2 2 2 2 2
39792024-02-03T17:24:15.255Z11/30 Write block 535 len 24576 data: 4 4 4 4 4 4
39802024-02-03T17:24:15.255Z12/30 Flush
39812024-02-03T17:24:15.260Z13/30 Write block 249 len 12288 data: 3 4 3
39822024-02-03T17:24:15.263Z14/30 Write block 584 len 28672 data: 4 3 4 3 3 4 4
39832024-02-03T17:24:15.263Z15/30 Write block 486 len 40960 data: 2 4 4 3 3 3 3 3 3 3
39842024-02-03T17:24:15.266Z16/30 Write block 187 len 40960 data: 5 4 5 4 4 2 3 2 2 2
39852024-02-03T17:24:15.267Z17/30 Write block 319 len 36864 data: 2 2 2 3 3 3 4 4 4
39862024-02-03T17:24:15.269Z18/30 Write block 2 len 36864 data: 3 3 4 4 4 5 6 5 3
39872024-02-03T17:24:15.269Z19/30 Read block 376 len 12288
39882024-02-03T17:24:15.272Z20/30 Write block 136 len 40960 data: 6 6 6 6 4 4 3 2 2 3
39892024-02-03T17:24:15.272Z21/30 Read block 261 len 16384
39902024-02-03T17:24:15.276Z22/30 Write block 288 len 24576 data: 4 4 4 5 6 6
39912024-02-03T17:24:15.276Z23/30 Read block 354 len 24576
39922024-02-03T17:24:15.281Z24/30 Write block 500 len 32768 data: 4 6 6 6 5 6 8 6
39932024-02-03T17:24:15.284Z25/30 Write block 563 len 16384 data: 3 4 5 6
39942024-02-03T17:24:15.284Z26/30 Flush
39952024-02-03T17:24:15.287Z27/30 Write block 207 len 4096 data: 4
39962024-02-03T17:24:15.288Z28/30 Read block 126 len 12288
39972024-02-03T17:24:15.288Z29/30 Write block 456 len 8192 data: 5 5
39982024-02-03T17:24:15.291Z30/30 Read block 263 len 40960
39992024-02-03T17:24:15.291Z----------------------------------------------------------------
40002024-02-03T17:24:15.291Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:18
40012024-02-03T17:24:15.291ZStates: Active Active Active
40022024-02-03T17:24:15.291ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40032024-02-03T17:24:15.291Z 13 Acked 1012 Write 3 Done Done Done false
40042024-02-03T17:24:15.291Z 14 Acked 1013 Write 7 Sent Done Done false
40052024-02-03T17:24:15.291Z 15 Acked 1014 Write 10 Sent Done Done false
40062024-02-03T17:24:15.291Z 16 Acked 1015 Write 10 Sent Done Done false
40072024-02-03T17:24:15.292Z 17 Acked 1016 Write 9 Sent Done Done false
40082024-02-03T17:24:15.292Z 18 Acked 1017 Write 9 Sent Done Done false
40092024-02-03T17:24:15.292Z 19 Acked 1018 Read 3 Sent Done Done false
40102024-02-03T17:24:15.292Z 20 Acked 1019 Write 10 Sent Done Done false
40112024-02-03T17:24:15.292Z 21 Acked 1020 Read 4 Sent Done Done false
40122024-02-03T17:24:15.292Z 22 Acked 1021 Write 6 Sent Done Done false
40132024-02-03T17:24:15.292Z 23 Acked 1022 Read 6 Sent Done Done false
40142024-02-03T17:24:15.292Z 24 Acked 1023 Write 8 Sent Done Done false
40152024-02-03T17:24:15.292Z 25 Acked 1024 Write 4 Sent Done Done false
40162024-02-03T17:24:15.292Z 26 Acked 1025 Flush 0 Sent Done Done false
40172024-02-03T17:24:15.292Z 27 Acked 1026 Write 1 Sent Done Done false
40182024-02-03T17:24:15.292Z 28 Acked 1027 Read 3 Sent Done Done false
40192024-02-03T17:24:15.292Z 29 Acked 1028 Write 2 Sent Done Done false
40202024-02-03T17:24:15.292Z 30 Acked 1029 Read 10 Sent Done Sent false
40212024-02-03T17:24:15.292Z STATES DS:0 DS:1 DS:2 TOTAL
40222024-02-03T17:24:15.292Z New 0 0 0 0
40232024-02-03T17:24:15.292Z Sent 17 0 1 18
40242024-02-03T17:24:15.292Z Done 1 18 17 36
40252024-02-03T17:24:15.292Z Skipped 0 0 0 0
40262024-02-03T17:24:15.292Z Error 0 0 0 0
40272024-02-03T17:24:15.292ZLast Flush: 1011 1025 1025
40282024-02-03T17:24:15.292ZDownstairs last five completed: 1011 1010 1009 1008 1007
40292024-02-03T17:24:15.292ZUpstairs last five completed: 30 29 28 27 26
40302024-02-03T17:24:15.292Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:14.450132089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1276}
40312024-02-03T17:24:15.293ZWrote out file "/tmp/test_repair_verify.data"
40322024-02-03T17:24:15.298Z
40332024-02-03T17:24:15.306ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40342024-02-03T17:24:15.332ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40352024-02-03T17:24:15.332Z 0 000-019 34 36 36 62 65 65 F F F <---
40362024-02-03T17:24:15.332Z 1 020-039 32 32 32 59 59 59 F F F
40372024-02-03T17:24:15.332Z 2 040-059 34 34 34 62 62 62 F F F
40382024-02-03T17:24:15.332Z 3 060-079 34 34 34 62 62 62 F F F
40392024-02-03T17:24:15.336Z 4 080-099 36 36 36 63 63 63 F F F
40402024-02-03T17:24:15.336Z 5 100-119 26 26 26 45 45 45 F F F
40412024-02-03T17:24:15.336Z 6 120-139 36 36 36 64 65 65 F F F <---
40422024-02-03T17:24:15.336Z 7 140-159 36 36 36 64 65 65 F F F <---
40432024-02-03T17:24:15.336Z 8 160-179 34 34 34 62 62 62 F F F
40442024-02-03T17:24:15.336Z 9 180-199 32 36 36 57 65 65 F F F <---
40452024-02-03T17:24:15.336Z 10 200-219 34 34 34 62 62 62 F T T <---
40462024-02-03T17:24:15.336Z 11 220-239 34 34 34 62 62 62 F F F
40472024-02-03T17:24:15.336Z 12 240-259 34 36 36 62 65 65 T F F <---
40482024-02-03T17:24:15.336Z 13 260-279 30 30 30 54 54 54 F F F
40492024-02-03T17:24:15.336Z 14 280-299 34 36 36 62 65 65 F F F <---
40502024-02-03T17:24:15.336Z 15 300-319 36 36 36 63 65 65 F F F <---
40512024-02-03T17:24:15.336Z 16 320-339 33 36 36 61 65 65 F F F <---
40522024-02-03T17:24:15.336Z 17 340-359 34 34 34 62 62 62 F F F
40532024-02-03T17:24:15.336Z 18 360-379 31 31 31 55 55 55 F F F
40542024-02-03T17:24:15.336Z 19 380-399 34 34 34 62 62 62 F F F
40552024-02-03T17:24:15.336Z 20 400-419 34 34 34 62 62 62 F F F
40562024-02-03T17:24:15.336Z 21 420-439 34 34 34 62 62 62 F F F
40572024-02-03T17:24:15.336Z 22 440-459 31 31 31 55 55 55 F T T <---
40582024-02-03T17:24:15.336Z 23 460-479 31 31 31 55 55 55 F F F
40592024-02-03T17:24:15.336Z 24 480-499 36 36 36 64 65 65 F F F <---
40602024-02-03T17:24:15.336Z 25 500-519 32 36 36 60 65 65 F F F <---
40612024-02-03T17:24:15.337Z 26 520-539 36 36 36 64 64 64 F F F
40622024-02-03T17:24:15.337Z 27 540-559 36 36 36 64 64 64 F F F
40632024-02-03T17:24:15.337Z 28 560-579 32 36 36 60 65 65 F F F <---
40642024-02-03T17:24:15.337Z 29 580-599 32 36 36 58 65 65 F F F <---
40652024-02-03T17:24:15.337ZMax gen: 36, Max flush: 65
40662024-02-03T17:24:15.337ZError: Difference in extent metadata found!
40672024-02-03T17:24:15.340ZA Difference in extent metadata is expected here
40682024-02-03T17:24:15.340ZOn loop 18
40692024-02-03T17:24:15.340Z
40702024-02-03T17:24:15.343ZVerifying data now
40712024-02-03T17:24:20.577ZNo change after verify
40722024-02-03T17:24:20.577ZLoop: 18 Downstairs dump after verify (and repair):
40732024-02-03T17:24:20.602ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40742024-02-03T17:24:20.602Z 0 000-019 36 36 36 65 65 65 F F F
40752024-02-03T17:24:20.602Z 1 020-039 32 32 32 59 59 59 F F F
40762024-02-03T17:24:20.602Z 2 040-059 34 34 34 62 62 62 F F F
40772024-02-03T17:24:20.602Z 3 060-079 34 34 34 62 62 62 F F F
40782024-02-03T17:24:20.605Z 4 080-099 36 36 36 63 63 63 F F F
40792024-02-03T17:24:20.606Z 5 100-119 26 26 26 45 45 45 F F F
40802024-02-03T17:24:20.606Z 6 120-139 36 36 36 65 65 65 F F F
40812024-02-03T17:24:20.606Z 7 140-159 36 36 36 65 65 65 F F F
40822024-02-03T17:24:20.606Z 8 160-179 34 34 34 62 62 62 F F F
40832024-02-03T17:24:20.606Z 9 180-199 36 36 36 65 65 65 F F F
40842024-02-03T17:24:20.606Z 10 200-219 37 37 37 66 66 66 F F F
40852024-02-03T17:24:20.606Z 11 220-239 34 34 34 62 62 62 F F F
40862024-02-03T17:24:20.606Z 12 240-259 36 36 36 65 65 65 F F F
40872024-02-03T17:24:20.606Z 13 260-279 30 30 30 54 54 54 F F F
40882024-02-03T17:24:20.606Z 14 280-299 36 36 36 65 65 65 F F F
40892024-02-03T17:24:20.606Z 15 300-319 36 36 36 65 65 65 F F F
40902024-02-03T17:24:20.606Z 16 320-339 36 36 36 65 65 65 F F F
40912024-02-03T17:24:20.607Z 17 340-359 34 34 34 62 62 62 F F F
40922024-02-03T17:24:20.607Z 18 360-379 31 31 31 55 55 55 F F F
40932024-02-03T17:24:20.607Z 19 380-399 34 34 34 62 62 62 F F F
40942024-02-03T17:24:20.607Z 20 400-419 34 34 34 62 62 62 F F F
40952024-02-03T17:24:20.607Z 21 420-439 34 34 34 62 62 62 F F F
40962024-02-03T17:24:20.607Z 22 440-459 37 37 37 66 66 66 F F F
40972024-02-03T17:24:20.607Z 23 460-479 31 31 31 55 55 55 F F F
40982024-02-03T17:24:20.607Z 24 480-499 36 36 36 65 65 65 F F F
40992024-02-03T17:24:20.607Z 25 500-519 36 36 36 65 65 65 F F F
41002024-02-03T17:24:20.607Z 26 520-539 36 36 36 64 64 64 F F F
41012024-02-03T17:24:20.607Z 27 540-559 36 36 36 64 64 64 F F F
41022024-02-03T17:24:20.607Z 28 560-579 36 36 36 65 65 65 F F F
41032024-02-03T17:24:20.607Z 29 580-599 36 36 36 65 65 65 F F F
41042024-02-03T17:24:20.608ZMax gen: 37, Max flush: 66
41052024-02-03T17:24:20.611Z
41062024-02-03T17:24:20.611ZBegin loop 19
41072024-02-03T17:24:20.611ZDownstairs to restart: 1
41082024-02-03T17:24:20.644Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.803536967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284}
41092024-02-03T17:24:20.648Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804029341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284}
41102024-02-03T17:24:20.648Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804174043Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284}
41112024-02-03T17:24:20.649Z{"msg":"Crucible 51dca4bf-6d1d-4172-83ef-938bbbc2b751 has session id: 3880cbfb-3d7a-4540-aa83-66858157f815","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804214109Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41122024-02-03T17:24:20.649Z{"msg":"Upstairs opts: Upstairs UUID: 51dca4bf-6d1d-4172-83ef-938bbbc2b751, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804297398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41132024-02-03T17:24:20.649Z{"msg":"Crucible stats registered with UUID: 51dca4bf-6d1d-4172-83ef-938bbbc2b751","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804356966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41142024-02-03T17:24:20.649ZCrucible runtime is spawned
41152024-02-03T17:24:20.649Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804595113Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284}
41162024-02-03T17:24:20.649Z{"msg":"51dca4bf-6d1d-4172-83ef-938bbbc2b751 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804702342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41172024-02-03T17:24:20.649Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804792286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41182024-02-03T17:24:20.649Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804837834Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41192024-02-03T17:24:20.649Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804942716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41202024-02-03T17:24:20.649Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.804987491Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41212024-02-03T17:24:20.649Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.80515588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41222024-02-03T17:24:20.649Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.805252339Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41232024-02-03T17:24:20.650Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.805303161Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41242024-02-03T17:24:20.650Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.80534676Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"io task","client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41252024-02-03T17:24:20.650Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.805389408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"io task","client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41262024-02-03T17:24:20.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.805433879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"io task","client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41272024-02-03T17:24:20.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.805526886Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"io task","client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41282024-02-03T17:24:20.650Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.80557526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"io task","client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41292024-02-03T17:24:20.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.806690307Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41302024-02-03T17:24:20.650Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.806765037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41312024-02-03T17:24:20.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.806809743Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41322024-02-03T17:24:20.650Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.806851657Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41332024-02-03T17:24:20.655Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.807685045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41342024-02-03T17:24:20.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.808021299Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41352024-02-03T17:24:20.655Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.808108375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41362024-02-03T17:24:20.655Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.808186958Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41372024-02-03T17:24:20.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.808720882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41382024-02-03T17:24:20.655Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.808782442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41392024-02-03T17:24:20.656Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.80880876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41402024-02-03T17:24:20.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.80912098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41412024-02-03T17:24:20.656Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.809187449Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41422024-02-03T17:24:20.656Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.809215525Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41432024-02-03T17:24:20.656Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.809242917Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41442024-02-03T17:24:20.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.809973654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41452024-02-03T17:24:20.656Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.81010507Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41462024-02-03T17:24:20.656Z{"msg":"[0]R flush_numbers[0..12]: [65, 59, 62, 62, 63, 45, 65, 65, 62, 65, 66, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810164951Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41472024-02-03T17:24:20.656Z{"msg":"[0]R generation[0..12]: [36, 32, 34, 34, 36, 26, 36, 36, 34, 36, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810193393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41482024-02-03T17:24:20.656Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810236776Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41492024-02-03T17:24:20.656Z{"msg":"[1]R flush_numbers[0..12]: [65, 59, 62, 62, 63, 45, 65, 65, 62, 65, 66, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810261969Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41502024-02-03T17:24:20.656Z{"msg":"[1]R generation[0..12]: [36, 32, 34, 34, 36, 26, 36, 36, 34, 36, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.81028654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41512024-02-03T17:24:20.656Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.81031079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41522024-02-03T17:24:20.657ZWait for a query_work_queue command to finish before sending IO
41532024-02-03T17:24:20.657Z{"msg":"[2]R flush_numbers[0..12]: [65, 59, 62, 62, 63, 45, 65, 65, 62, 65, 66, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810354524Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41542024-02-03T17:24:20.657Z{"msg":"[2]R generation[0..12]: [36, 32, 34, 34, 36, 26, 36, 36, 34, 36, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810379914Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41552024-02-03T17:24:20.657Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810422752Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41562024-02-03T17:24:20.657Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810446543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41572024-02-03T17:24:20.657Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810468715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41582024-02-03T17:24:20.657Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810489889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41592024-02-03T17:24:20.657Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.81053758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"":"downstairs","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41602024-02-03T17:24:20.657Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810561995Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41612024-02-03T17:24:20.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810609327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"0","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41622024-02-03T17:24:20.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810633025Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"1","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41632024-02-03T17:24:20.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810654628Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"client":"2","":"downstairs","session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41642024-02-03T17:24:20.657Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810676609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41652024-02-03T17:24:20.658Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810698725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41662024-02-03T17:24:20.658Z{"msg":"51dca4bf-6d1d-4172-83ef-938bbbc2b751 is now active with session: 3880cbfb-3d7a-4540-aa83-66858157f815","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810720385Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41672024-02-03T17:24:20.658Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810739851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284,"session_id":"3880cbfb-3d7a-4540-aa83-66858157f815"}
41682024-02-03T17:24:20.658Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:19.810760414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284}
41692024-02-03T17:24:20.658ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41702024-02-03T17:24:20.658ZLoading write count information from file "/tmp/test_repair_verify.data"
41712024-02-03T17:24:20.658ZRun Repair workload
41722024-02-03T17:24:20.658Z01/30 Write block 443 len 4096 data: 5
41732024-02-03T17:24:20.662Z02/30 Write block 169 len 20480 data: 5 5 4 3 3
41742024-02-03T17:24:20.662Z03/30 Write block 66 len 24576 data: 3 2 2 3 2 2
41752024-02-03T17:24:20.662Z04/30 Write block 247 len 16384 data: 3 3 4 5
41762024-02-03T17:24:20.662Z05/30 Write block 308 len 16384 data: 3 3 3 2
41772024-02-03T17:24:20.662Z06/30 Read block 576 len 16384
41782024-02-03T17:24:20.663Z07/30 Read block 469 len 16384
41792024-02-03T17:24:20.663Z08/30 Read block 208 len 28672
41802024-02-03T17:24:20.666Z09/30 Write block 306 len 4096 data: 4
41812024-02-03T17:24:20.667Z10/30 Read block 302 len 24576
41822024-02-03T17:24:20.667Z11/30 Write block 590 len 36864 data: 5 6 6 6 6 6 5 5 5
41832024-02-03T17:24:20.670Z12/30 Read block 468 len 12288
41842024-02-03T17:24:20.670Z13/30 Read block 447 len 28672
41852024-02-03T17:24:20.674Z14/30 Write block 46 len 36864 data: 3 3 3 5 6 6 6 5 3
41862024-02-03T17:24:20.674Z15/30 Flush
41872024-02-03T17:24:20.677Z16/30 Write block 246 len 24576 data: 3 4 4 5 6 4
41882024-02-03T17:24:20.677Z17/30 Write block 129 len 4096 data: 5
41892024-02-03T17:24:20.682Z18/30 Read block 583 len 20480
41902024-02-03T17:24:20.682Z19/30 Read block 323 len 4096
41912024-02-03T17:24:20.682Z20/30 Read block 153 len 32768
41922024-02-03T17:24:20.685Z21/30 Write block 75 len 40960 data: 3 3 3 3 2 2 2 3 4 5
41932024-02-03T17:24:20.686Z22/30 Flush
41942024-02-03T17:24:20.689Z23/30 Write block 111 len 36864 data: 5 5 5 6 5 4 6 6 5
41952024-02-03T17:24:20.689Z24/30 Flush
41962024-02-03T17:24:20.692Z25/30 Write block 590 len 4096 data: 6
41972024-02-03T17:24:20.692Z26/30 Read block 321 len 36864
41982024-02-03T17:24:20.707Z27/30 Write block 477 len 24576 data: 5 5 5 4 3 3
41992024-02-03T17:24:20.712Z28/30 Write block 360 len 24576 data: 5 5 5 5 5 6
42002024-02-03T17:24:20.712Z29/30 Read block 518 len 40960
42012024-02-03T17:24:20.712Z30/30 Write block 156 len 40960 data: 5 5 3 3 3 4 5 4 5 5
42022024-02-03T17:24:20.712Z----------------------------------------------------------------
42032024-02-03T17:24:20.712Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
42042024-02-03T17:24:20.712ZStates: Active Active Active
42052024-02-03T17:24:20.712ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42062024-02-03T17:24:20.712Z 1 Acked 1000 Write 1 Done Sent Done false
42072024-02-03T17:24:20.712Z 2 Acked 1001 Write 5 Done Sent Done false
42082024-02-03T17:24:20.712Z 3 Acked 1002 Write 6 Done Sent Done false
42092024-02-03T17:24:20.712Z 4 Acked 1003 Write 4 Done Sent Done false
42102024-02-03T17:24:20.712Z 5 Acked 1004 Write 4 Done Sent Done false
42112024-02-03T17:24:20.713Z 6 Acked 1005 Read 4 Done Sent Done false
42122024-02-03T17:24:20.713Z 7 Acked 1006 Read 4 Done Sent Done false
42132024-02-03T17:24:20.713Z 8 Acked 1007 Read 7 Done Sent Done false
42142024-02-03T17:24:20.713Z 9 Acked 1008 Write 1 Done Sent Done false
42152024-02-03T17:24:20.713Z 10 Acked 1009 Read 6 Done Sent Done false
42162024-02-03T17:24:20.713Z 11 Acked 1010 Write 9 Done Sent Done false
42172024-02-03T17:24:20.713Z 12 Acked 1011 Read 3 Done Sent Done false
42182024-02-03T17:24:20.713Z 13 Acked 1012 Read 7 Done Sent Done false
42192024-02-03T17:24:20.713Z 14 Acked 1013 Write 9 Done Sent Done false
42202024-02-03T17:24:20.713Z 15 Acked 1014 Flush 0 Done Sent Done false
42212024-02-03T17:24:20.713Z 16 Acked 1015 Write 6 Done Sent Done false
42222024-02-03T17:24:20.713Z 17 Acked 1016 Write 1 Done Sent Done false
42232024-02-03T17:24:20.714Z 18 Acked 1017 Read 5 Done Sent Done false
42242024-02-03T17:24:20.714Z 19 Acked 1018 Read 1 Done Sent Done false
42252024-02-03T17:24:20.714Z 20 Acked 1019 Read 8 Done Sent Done false
42262024-02-03T17:24:20.714Z 21 Acked 1020 Write 10 Done Sent Done false
42272024-02-03T17:24:20.714Z 22 Acked 1021 Flush 0 Done Sent Done false
42282024-02-03T17:24:20.714Z 23 Acked 1022 Write 9 Done Sent Done false
42292024-02-03T17:24:20.714Z 24 Acked 1023 Flush 0 Done Sent Done false
42302024-02-03T17:24:20.714Z 25 Acked 1024 Write 1 Done Sent Done false
42312024-02-03T17:24:20.714Z 26 Acked 1025 Read 9 Done Sent Done false
42322024-02-03T17:24:20.714Z 27 Acked 1026 Write 6 Done Sent Done false
42332024-02-03T17:24:20.714Z 28 Acked 1027 Write 6 Done Sent Done false
42342024-02-03T17:24:20.714Z 29 Acked 1028 Read 10 Done Sent Done false
42352024-02-03T17:24:20.714Z 30 Acked 1029 Write 10 Sent Sent Sent false
42362024-02-03T17:24:20.715Z STATES DS:0 DS:1 DS:2 TOTAL
42372024-02-03T17:24:20.715Z New 0 0 0 0
42382024-02-03T17:24:20.715Z Sent 1 30 1 32
42392024-02-03T17:24:20.715Z Done 29 0 29 58
42402024-02-03T17:24:20.715Z Skipped 0 0 0 0
42412024-02-03T17:24:20.715Z Error 0 0 0 0
42422024-02-03T17:24:20.715ZLast Flush: 1023 0 1023
42432024-02-03T17:24:20.715ZDownstairs last five completed:
42442024-02-03T17:24:20.715ZUpstairs last five completed: 30 29 28 27 26
42452024-02-03T17:24:20.715Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:19.860756858Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1284}
42462024-02-03T17:24:20.715ZWrote out file "/tmp/test_repair_verify.data"
42472024-02-03T17:24:20.715Z
42482024-02-03T17:24:20.719ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42492024-02-03T17:24:20.749ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42502024-02-03T17:24:20.749Z 0 000-019 36 36 36 65 65 65 F F F
42512024-02-03T17:24:20.753Z 1 020-039 32 32 32 59 59 59 F F F
42522024-02-03T17:24:20.753Z 2 040-059 38 34 38 67 62 67 F F F <---
42532024-02-03T17:24:20.753Z 3 060-079 38 34 38 68 62 68 F F F <---
42542024-02-03T17:24:20.753Z 4 080-099 38 36 38 68 63 68 F F F <---
42552024-02-03T17:24:20.753Z 5 100-119 38 26 38 69 45 69 F F F <---
42562024-02-03T17:24:20.754Z 6 120-139 38 36 38 68 65 68 F F F <---
42572024-02-03T17:24:20.754Z 7 140-159 36 36 36 65 65 65 T F T <---
42582024-02-03T17:24:20.754Z 8 160-179 38 34 38 67 62 67 T F T <---
42592024-02-03T17:24:20.754Z 9 180-199 36 36 36 65 65 65 F F F
42602024-02-03T17:24:20.754Z 10 200-219 37 37 37 66 66 66 F F F
42612024-02-03T17:24:20.754Z 11 220-239 34 34 34 62 62 62 F F F
42622024-02-03T17:24:20.754Z 12 240-259 38 36 38 68 65 68 F F F <---
42632024-02-03T17:24:20.754Z 13 260-279 30 30 30 54 54 54 F F F
42642024-02-03T17:24:20.754Z 14 280-299 36 36 36 65 65 65 F F F
42652024-02-03T17:24:20.754Z 15 300-319 38 36 38 67 65 67 F F F <---
42662024-02-03T17:24:20.754Z 16 320-339 36 36 36 65 65 65 F F F
42672024-02-03T17:24:20.754Z 17 340-359 34 34 34 62 62 62 F F F
42682024-02-03T17:24:20.754Z 18 360-379 31 31 31 55 55 55 T F T <---
42692024-02-03T17:24:20.755Z 19 380-399 34 34 34 62 62 62 F F F
42702024-02-03T17:24:20.755Z 20 400-419 34 34 34 62 62 62 F F F
42712024-02-03T17:24:20.755Z 21 420-439 34 34 34 62 62 62 F F F
42722024-02-03T17:24:20.755Z 22 440-459 38 37 38 67 66 67 F F F <---
42732024-02-03T17:24:20.755Z 23 460-479 31 31 31 55 55 55 T F T <---
42742024-02-03T17:24:20.755Z 24 480-499 36 36 36 65 65 65 T F T <---
42752024-02-03T17:24:20.755Z 25 500-519 36 36 36 65 65 65 F F F
42762024-02-03T17:24:20.755Z 26 520-539 36 36 36 64 64 64 F F F
42772024-02-03T17:24:20.755Z 27 540-559 36 36 36 64 64 64 F F F
42782024-02-03T17:24:20.755Z 28 560-579 36 36 36 65 65 65 F F F
42792024-02-03T17:24:20.755Z 29 580-599 38 36 38 67 65 67 T F T <---
42802024-02-03T17:24:20.755ZMax gen: 38, Max flush: 69
42812024-02-03T17:24:20.755ZError: Difference in extent metadata found!
42822024-02-03T17:24:20.756ZA Difference in extent metadata is expected here
42832024-02-03T17:24:20.756ZOn loop 19
42842024-02-03T17:24:20.756Z
42852024-02-03T17:24:20.762ZVerifying data now
42862024-02-03T17:24:25.965ZNo change after verify
42872024-02-03T17:24:25.965ZLoop: 19 Downstairs dump after verify (and repair):
42882024-02-03T17:24:25.989ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42892024-02-03T17:24:25.989Z 0 000-019 36 36 36 65 65 65 F F F
42902024-02-03T17:24:25.993Z 1 020-039 32 32 32 59 59 59 F F F
42912024-02-03T17:24:25.993Z 2 040-059 38 38 38 67 67 67 F F F
42922024-02-03T17:24:25.993Z 3 060-079 38 38 38 68 68 68 F F F
42932024-02-03T17:24:25.993Z 4 080-099 38 38 38 68 68 68 F F F
42942024-02-03T17:24:25.993Z 5 100-119 38 38 38 69 69 69 F F F
42952024-02-03T17:24:25.993Z 6 120-139 38 38 38 68 68 68 F F F
42962024-02-03T17:24:25.993Z 7 140-159 39 39 39 70 70 70 F F F
42972024-02-03T17:24:25.993Z 8 160-179 39 39 39 70 70 70 F F F
42982024-02-03T17:24:25.993Z 9 180-199 36 36 36 65 65 65 F F F
42992024-02-03T17:24:25.993Z 10 200-219 37 37 37 66 66 66 F F F
43002024-02-03T17:24:25.993Z 11 220-239 34 34 34 62 62 62 F F F
43012024-02-03T17:24:25.993Z 12 240-259 38 38 38 68 68 68 F F F
43022024-02-03T17:24:25.993Z 13 260-279 30 30 30 54 54 54 F F F
43032024-02-03T17:24:25.993Z 14 280-299 36 36 36 65 65 65 F F F
43042024-02-03T17:24:25.993Z 15 300-319 38 38 38 67 67 67 F F F
43052024-02-03T17:24:25.993Z 16 320-339 36 36 36 65 65 65 F F F
43062024-02-03T17:24:25.993Z 17 340-359 34 34 34 62 62 62 F F F
43072024-02-03T17:24:25.993Z 18 360-379 39 39 39 70 70 70 F F F
43082024-02-03T17:24:25.993Z 19 380-399 34 34 34 62 62 62 F F F
43092024-02-03T17:24:25.993Z 20 400-419 34 34 34 62 62 62 F F F
43102024-02-03T17:24:25.993Z 21 420-439 34 34 34 62 62 62 F F F
43112024-02-03T17:24:25.993Z 22 440-459 38 38 38 67 67 67 F F F
43122024-02-03T17:24:25.994Z 23 460-479 39 39 39 70 70 70 F F F
43132024-02-03T17:24:25.994Z 24 480-499 39 39 39 70 70 70 F F F
43142024-02-03T17:24:25.994Z 25 500-519 36 36 36 65 65 65 F F F
43152024-02-03T17:24:25.994Z 26 520-539 36 36 36 64 64 64 F F F
43162024-02-03T17:24:25.994Z 27 540-559 36 36 36 64 64 64 F F F
43172024-02-03T17:24:25.994Z 28 560-579 36 36 36 65 65 65 F F F
43182024-02-03T17:24:25.994Z 29 580-599 39 39 39 70 70 70 F F F
43192024-02-03T17:24:25.994ZMax gen: 39, Max flush: 70
43202024-02-03T17:24:25.997Z
43212024-02-03T17:24:25.997ZBegin loop 20
43222024-02-03T17:24:25.997ZDownstairs to restart: 0
43232024-02-03T17:24:26.028Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.186980286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292}
43242024-02-03T17:24:26.028Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.187604392Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292}
43252024-02-03T17:24:26.028Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.187637962Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292}
43262024-02-03T17:24:26.032ZCrucible runtime is spawned
43272024-02-03T17:24:26.032Z{"msg":"Crucible d0323df0-0f44-4fa2-a230-285769f0b895 has session id: 3d2bb798-46d1-431a-a76e-20b074952dee","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.187810022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43282024-02-03T17:24:26.032Z{"msg":"Upstairs opts: Upstairs UUID: d0323df0-0f44-4fa2-a230-285769f0b895, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.187906824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43292024-02-03T17:24:26.032Z{"msg":"Crucible stats registered with UUID: d0323df0-0f44-4fa2-a230-285769f0b895","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.187936871Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43302024-02-03T17:24:26.032Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.187978348Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292}
43312024-02-03T17:24:26.032Z{"msg":"d0323df0-0f44-4fa2-a230-285769f0b895 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188187744Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43322024-02-03T17:24:26.032Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188241919Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43332024-02-03T17:24:26.032Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.18826896Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43342024-02-03T17:24:26.032Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188341112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43352024-02-03T17:24:26.032Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188373264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43362024-02-03T17:24:26.032Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188450964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43372024-02-03T17:24:26.032Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188493185Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43382024-02-03T17:24:26.032Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188531565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"io task","client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43392024-02-03T17:24:26.032Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188574007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"io task","client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43402024-02-03T17:24:26.032Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188643919Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"io task","client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43412024-02-03T17:24:26.032Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.18871767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"io task","client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43422024-02-03T17:24:26.032Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.18878732Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"io task","client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43432024-02-03T17:24:26.032Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.188882853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"io task","client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43442024-02-03T17:24:26.032Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.190177428Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43452024-02-03T17:24:26.032Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.190276838Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43462024-02-03T17:24:26.033Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.190320031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43472024-02-03T17:24:26.033Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.190361786Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43482024-02-03T17:24:26.033Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.191000493Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43492024-02-03T17:24:26.033Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.191088119Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43502024-02-03T17:24:26.033Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.191136404Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43512024-02-03T17:24:26.033Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.191224745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43522024-02-03T17:24:26.045Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.192154973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43532024-02-03T17:24:26.045Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.192220893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43542024-02-03T17:24:26.045Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.192248381Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43552024-02-03T17:24:26.045Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.192272171Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43562024-02-03T17:24:26.045Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.192323956Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43572024-02-03T17:24:26.045Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.192349792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43582024-02-03T17:24:26.045Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.19237262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43592024-02-03T17:24:26.046Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193215812Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43602024-02-03T17:24:26.046Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193285863Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43612024-02-03T17:24:26.046Z{"msg":"[0]R flush_numbers[0..12]: [65, 59, 67, 68, 68, 69, 68, 70, 70, 65, 66, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193315442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43622024-02-03T17:24:26.046Z{"msg":"[0]R generation[0..12]: [36, 32, 38, 38, 38, 38, 38, 39, 39, 36, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193359066Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43632024-02-03T17:24:26.046Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193386292Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43642024-02-03T17:24:26.046Z{"msg":"[1]R flush_numbers[0..12]: [65, 59, 67, 68, 68, 69, 68, 70, 70, 65, 66, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.19342854Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43652024-02-03T17:24:26.046Z{"msg":"[1]R generation[0..12]: [36, 32, 38, 38, 38, 38, 38, 39, 39, 36, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193454454Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43662024-02-03T17:24:26.046Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
43672024-02-03T17:24:26.046Z,"time":"2024-02-03T17:24:25.193497145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43682024-02-03T17:24:26.046Z{"msg":"[2]R flush_numbers[0..12]: [65, 59, 67, 68, 68, 69, 68, 70, 70, 65, 66, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193630777Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43692024-02-03T17:24:26.046Z{"msg":"[2]R generation[0..12]: [36, 32, 38, 38, 38, 38, 38, 39, 39, 36, 37, 34]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193659196Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43702024-02-03T17:24:26.046Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193689104Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43712024-02-03T17:24:26.046Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193750261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43722024-02-03T17:24:26.046Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.19377837Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43732024-02-03T17:24:26.046Z{"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193835345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43742024-02-03T17:24:26.046Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193872698Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"":"downstairs","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43752024-02-03T17:24:26.046Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.193939322Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43762024-02-03T17:24:26.046Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.194006304Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"0","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43772024-02-03T17:24:26.046Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.194034157Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"1","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43782024-02-03T17:24:26.046Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.19412036Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"client":"2","":"downstairs","session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43792024-02-03T17:24:26.046Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43802024-02-03T17:24:26.046Z,"time":"2024-02-03T17:24:25.194192957Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43812024-02-03T17:24:26.046Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.194229066Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43822024-02-03T17:24:26.047Z{"msg":"d0323df0-0f44-4fa2-a230-285769f0b895 is now active with session: 3d2bb798-46d1-431a-a76e-20b074952dee","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.194288242Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43832024-02-03T17:24:26.047Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.194313179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292,"session_id":"3d2bb798-46d1-431a-a76e-20b074952dee"}
43842024-02-03T17:24:26.047Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:25.194353918Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292}
43852024-02-03T17:24:26.047ZLoading write count information from file "/tmp/test_repair_verify.data"
43862024-02-03T17:24:26.047ZRun Repair workload
43872024-02-03T17:24:26.047Z01/30 Write block 567 len 4096 data: 6
43882024-02-03T17:24:26.050Z02/30 Write block 353 len 12288 data: 4 3 2
43892024-02-03T17:24:26.050Z03/30 Write block 582 len 32768 data: 6 5 5 4 5 4 4 5
43902024-02-03T17:24:26.050Z04/30 Write block 10 len 16384 data: 4 4 4 4
43912024-02-03T17:24:26.050Z05/30 Write block 134 len 20480 data: 5 5 7 7 7
43922024-02-03T17:24:26.050Z06/30 Read block 188 len 36864
43932024-02-03T17:24:26.051Z07/30 Flush
43942024-02-03T17:24:26.051Z08/30 Write block 83 len 28672 data: 5 6 4 3 5 4 5
43952024-02-03T17:24:26.051Z09/30 Read block 434 len 20480
43962024-02-03T17:24:26.054Z10/30 Write block 132 len 4096 data: 5
43972024-02-03T17:24:26.054Z11/30 Flush
43982024-02-03T17:24:26.057Z12/30 Write block 287 len 4096 data: 4
43992024-02-03T17:24:26.057Z13/30 Read block 339 len 8192
44002024-02-03T17:24:26.057Z14/30 Read block 165 len 16384
44012024-02-03T17:24:26.060Z15/30 Write block 187 len 16384 data: 6 5 6 5
44022024-02-03T17:24:26.060Z16/30 Write block 129 len 16384 data: 6 6 6 6
44032024-02-03T17:24:26.063Z17/30 Write block 239 len 12288 data: 6 4 4
44042024-02-03T17:24:26.063Z18/30 Read block 266 len 16384
44052024-02-03T17:24:26.063Z19/30 Read block 1 len 24576
44062024-02-03T17:24:26.066Z20/30 Read block 176 len 4096
44072024-02-03T17:24:26.066Z21/30 Read block 270 len 40960
44082024-02-03T17:24:26.069Z22/30 Read block 165 len 32768
44092024-02-03T17:24:26.069Z23/30 Read block 305 len 24576
44102024-02-03T17:24:26.072Z24/30 Read block 583 len 40960
44112024-02-03T17:24:26.072Z25/30 Read block 516 len 8192
44122024-02-03T17:24:26.075Z26/30 Flush
44132024-02-03T17:24:26.080Z27/30 Write block 25 len 12288 data: 5 3 3
44142024-02-03T17:24:26.081Z28/30 Read block 251 len 12288
44152024-02-03T17:24:26.084Z29/30 Write block 125 len 32768 data: 6 6 5 5 7 7 7 7
44162024-02-03T17:24:26.084Z30/30 Write block 281 len 4096 data: 4
44172024-02-03T17:24:26.084Z----------------------------------------------------------------
44182024-02-03T17:24:26.084Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44192024-02-03T17:24:26.084ZStates: Active Active Active
44202024-02-03T17:24:26.084ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44212024-02-03T17:24:26.084Z 1 Acked 1000 Write 1 Done Done Done false
44222024-02-03T17:24:26.084Z 2 Acked 1001 Write 3 Done Done Done false
44232024-02-03T17:24:26.085Z 3 Acked 1002 Write 8 Done Done Done false
44242024-02-03T17:24:26.085Z 4 Acked 1003 Write 4 Done Done Done false
44252024-02-03T17:24:26.085Z 5 Acked 1004 Write 5 Done Done Done false
44262024-02-03T17:24:26.085Z 6 Acked 1005 Read 9 Sent Done Done false
44272024-02-03T17:24:26.085Z 7 Acked 1006 Flush 0 Sent Done Done false
44282024-02-03T17:24:26.085Z 8 Acked 1007 Write 7 Sent Done Done false
44292024-02-03T17:24:26.085Z 9 Acked 1008 Read 5 Sent Done Done false
44302024-02-03T17:24:26.085Z 10 Acked 1009 Write 1 Sent Done Done false
44312024-02-03T17:24:26.085Z 11 Acked 1010 Flush 0 Sent Done Done false
44322024-02-03T17:24:26.085Z 12 Acked 1011 Write 1 Sent Done Done false
44332024-02-03T17:24:26.085Z 13 Acked 1012 Read 2 Sent Done Done false
44342024-02-03T17:24:26.085Z 14 Acked 1013 Read 4 Sent Done Done false
44352024-02-03T17:24:26.085Z 15 Acked 1014 Write 4 Sent Done Done false
44362024-02-03T17:24:26.085Z 16 Acked 1015 Write 4 Sent Done Done false
44372024-02-03T17:24:26.085Z 17 Acked 1016 Write 3 Sent Done Done false
44382024-02-03T17:24:26.085Z 18 Acked 1017 Read 4 Sent Done Done false
44392024-02-03T17:24:26.085Z 19 Acked 1018 Read 6 Sent Done Done false
44402024-02-03T17:24:26.085Z 20 Acked 1019 Read 1 Sent Done Done false
44412024-02-03T17:24:26.085Z 21 Acked 1020 Read 10 Sent Done Done false
44422024-02-03T17:24:26.085Z 22 Acked 1021 Read 8 Sent Done Done false
44432024-02-03T17:24:26.085Z 23 Acked 1022 Read 6 Sent Done Done false
44442024-02-03T17:24:26.085Z 24 Acked 1023 Read 10 Sent Done Done false
44452024-02-03T17:24:26.085Z 25 Acked 1024 Read 2 Sent Done Done false
44462024-02-03T17:24:26.085Z 26 Acked 1025 Flush 0 Sent Done Done false
44472024-02-03T17:24:26.085Z 27 Acked 1026 Write 3 Sent Done Done false
44482024-02-03T17:24:26.086Z 28 Acked 1027 Read 3 Sent Done Done false
44492024-02-03T17:24:26.086Z 29 Acked 1028 Write 8 Sent Sent Sent false
44502024-02-03T17:24:26.086Z 30 Acked 1029 Write 1 Sent Sent Sent false
44512024-02-03T17:24:26.086Z STATES DS:0 DS:1 DS:2 TOTAL
44522024-02-03T17:24:26.086Z New 0 0 0 0
44532024-02-03T17:24:26.086Z Sent 25 2 2 29
44542024-02-03T17:24:26.086Z Done 5 28 28 61
44552024-02-03T17:24:26.086Z Skipped 0 0 0 0
44562024-02-03T17:24:26.086Z Error 0 0 0 0
44572024-02-03T17:24:26.086ZLast Flush: 0 1025 1025
44582024-02-03T17:24:26.086ZDownstairs last five completed:
44592024-02-03T17:24:26.086ZUpstairs last five completed: 30 29 28 27 26
44602024-02-03T17:24:26.086Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:25.240196959Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1292}
44612024-02-03T17:24:26.086ZWrote out file "/tmp/test_repair_verify.data"
44622024-02-03T17:24:26.089Z
44632024-02-03T17:24:26.095ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44642024-02-03T17:24:26.123ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44652024-02-03T17:24:26.123Z 0 000-019 36 40 40 65 71 71 T F F <---
44662024-02-03T17:24:26.126Z 1 020-039 32 32 32 59 59 59 F T T <---
44672024-02-03T17:24:26.126Z 2 040-059 38 38 38 67 67 67 F F F
44682024-02-03T17:24:26.126Z 3 060-079 38 38 38 68 68 68 F F F
44692024-02-03T17:24:26.126Z 4 080-099 38 40 40 68 72 72 F F F <---
44702024-02-03T17:24:26.126Z 5 100-119 38 38 38 69 69 69 F F F
44712024-02-03T17:24:26.126Z 6 120-139 38 40 40 68 73 73 T T T <---
44722024-02-03T17:24:26.126Z 7 140-159 39 39 39 70 70 70 F F F
44732024-02-03T17:24:26.126Z 8 160-179 39 39 39 70 70 70 F F F
44742024-02-03T17:24:26.126Z 9 180-199 36 40 40 65 73 73 F F F <---
44752024-02-03T17:24:26.126Z 10 200-219 37 37 37 66 66 66 F F F
44762024-02-03T17:24:26.126Z 11 220-239 34 40 40 62 73 73 F F F <---
44772024-02-03T17:24:26.126Z 12 240-259 38 40 40 68 73 73 F F F <---
44782024-02-03T17:24:26.127Z 13 260-279 30 30 30 54 54 54 F F F
44792024-02-03T17:24:26.127Z 14 280-299 36 40 40 65 73 73 F T T <---
44802024-02-03T17:24:26.127Z 15 300-319 38 38 38 67 67 67 F F F
44812024-02-03T17:24:26.127Z 16 320-339 36 36 36 65 65 65 F F F
44822024-02-03T17:24:26.127Z 17 340-359 34 40 40 62 71 71 T F F <---
44832024-02-03T17:24:26.127Z 18 360-379 39 39 39 70 70 70 F F F
44842024-02-03T17:24:26.127Z 19 380-399 34 34 34 62 62 62 F F F
44852024-02-03T17:24:26.127Z 20 400-419 34 34 34 62 62 62 F F F
44862024-02-03T17:24:26.127Z 21 420-439 34 34 34 62 62 62 F F F
44872024-02-03T17:24:26.127Z 22 440-459 38 38 38 67 67 67 F F F
44882024-02-03T17:24:26.127Z 23 460-479 39 39 39 70 70 70 F F F
44892024-02-03T17:24:26.127Z 24 480-499 39 39 39 70 70 70 F F F
44902024-02-03T17:24:26.127Z 25 500-519 36 36 36 65 65 65 F F F
44912024-02-03T17:24:26.127Z 26 520-539 36 36 36 64 64 64 F F F
44922024-02-03T17:24:26.127Z 27 540-559 36 36 36 64 64 64 F F F
44932024-02-03T17:24:26.127Z 28 560-579 36 40 40 65 71 71 T F F <---
44942024-02-03T17:24:26.127Z 29 580-599 39 40 40 70 71 71 T F F <---
44952024-02-03T17:24:26.127ZMax gen: 40, Max flush: 73
44962024-02-03T17:24:26.127ZError: Difference in extent metadata found!
44972024-02-03T17:24:26.130ZA Difference in extent metadata is expected here
44982024-02-03T17:24:26.131ZOn loop 20
44992024-02-03T17:24:26.131Z
45002024-02-03T17:24:26.133ZVerifying data now
45012024-02-03T17:24:32.115ZNo change after verify
45022024-02-03T17:24:32.116ZLoop: 20 Downstairs dump after verify (and repair):
45032024-02-03T17:24:32.136ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45042024-02-03T17:24:32.136Z 0 000-019 40 40 40 71 71 71 F F F
45052024-02-03T17:24:32.140Z 1 020-039 41 41 41 74 74 74 F F F
45062024-02-03T17:24:32.140Z 2 040-059 38 38 38 67 67 67 F F F
45072024-02-03T17:24:32.140Z 3 060-079 38 38 38 68 68 68 F F F
45082024-02-03T17:24:32.140Z 4 080-099 40 40 40 72 72 72 F F F
45092024-02-03T17:24:32.140Z 5 100-119 38 38 38 69 69 69 F F F
45102024-02-03T17:24:32.141Z 6 120-139 41 41 41 74 74 74 F F F
45112024-02-03T17:24:32.141Z 7 140-159 39 39 39 70 70 70 F F F
45122024-02-03T17:24:32.141Z 8 160-179 39 39 39 70 70 70 F F F
45132024-02-03T17:24:32.141Z 9 180-199 40 40 40 73 73 73 F F F
45142024-02-03T17:24:32.141Z 10 200-219 37 37 37 66 66 66 F F F
45152024-02-03T17:24:32.141Z 11 220-239 40 40 40 73 73 73 F F F
45162024-02-03T17:24:32.141Z 12 240-259 40 40 40 73 73 73 F F F
45172024-02-03T17:24:32.141Z 13 260-279 30 30 30 54 54 54 F F F
45182024-02-03T17:24:32.141Z 14 280-299 41 41 41 74 74 74 F F F
45192024-02-03T17:24:32.141Z 15 300-319 38 38 38 67 67 67 F F F
45202024-02-03T17:24:32.141Z 16 320-339 36 36 36 65 65 65 F F F
45212024-02-03T17:24:32.141Z 17 340-359 40 40 40 71 71 71 F F F
45222024-02-03T17:24:32.142Z 18 360-379 39 39 39 70 70 70 F F F
45232024-02-03T17:24:32.142Z 19 380-399 34 34 34 62 62 62 F F F
45242024-02-03T17:24:32.142Z 20 400-419 34 34 34 62 62 62 F F F
45252024-02-03T17:24:32.142Z 21 420-439 34 34 34 62 62 62 F F F
45262024-02-03T17:24:32.142Z 22 440-459 38 38 38 67 67 67 F F F
45272024-02-03T17:24:32.142Z 23 460-479 39 39 39 70 70 70 F F F
45282024-02-03T17:24:32.142Z 24 480-499 39 39 39 70 70 70 F F F
45292024-02-03T17:24:32.142Z 25 500-519 36 36 36 65 65 65 F F F
45302024-02-03T17:24:32.142Z 26 520-539 36 36 36 64 64 64 F F F
45312024-02-03T17:24:32.142Z 27 540-559 36 36 36 64 64 64 F F F
45322024-02-03T17:24:32.142Z 28 560-579 40 40 40 71 71 71 F F F
45332024-02-03T17:24:32.142Z 29 580-599 40 40 40 71 71 71 F F F
45342024-02-03T17:24:32.142ZMax gen: 41, Max flush: 74
45352024-02-03T17:24:32.146Z
45362024-02-03T17:24:32.146ZBegin loop 21
45372024-02-03T17:24:32.146ZDownstairs to restart: 2
45382024-02-03T17:24:32.175Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.335149253Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300}
45392024-02-03T17:24:32.175Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.335593463Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300}
45402024-02-03T17:24:32.179Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
45412024-02-03T17:24:32.180Z,"time":"2024-02-03T17:24:31.335705198Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300}
45422024-02-03T17:24:32.180Z{"msg":"Crucible d724c70b-c21e-4304-b41c-fc6badbfb9ae has session id: cc03d09f-6004-43e5-9bbc-fbb107ca5587","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.335823164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45432024-02-03T17:24:32.180Z{"msg":"Upstairs opts: Upstairs UUID: d724c70b-c21e-4304-b41c-fc6badbfb9ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.335919047Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45442024-02-03T17:24:32.180Z{"msg":"Crucible stats registered with UUID: d724c70b-c21e-4304-b41c-fc6badbfb9ae","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.335975664Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45452024-02-03T17:24:32.180Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.335999699Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300}
45462024-02-03T17:24:32.180Z{"msg":"d724c70b-c21e-4304-b41c-fc6badbfb9ae active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336132926Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45472024-02-03T17:24:32.180Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336184518Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45482024-02-03T17:24:32.180Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336211057Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45492024-02-03T17:24:32.180Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336285262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45502024-02-03T17:24:32.180Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336329156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45512024-02-03T17:24:32.180Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336502182Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45522024-02-03T17:24:32.181Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336589429Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45532024-02-03T17:24:32.181Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336635593Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"io task","client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45542024-02-03T17:24:32.181Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336677835Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"io task","client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45552024-02-03T17:24:32.181Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336720205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"io task","client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45562024-02-03T17:24:32.181Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336782991Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"io task","client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45572024-02-03T17:24:32.181Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336809275Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"io task","client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45582024-02-03T17:24:32.181Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.336831497Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"io task","client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45592024-02-03T17:24:32.181Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.338009189Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45602024-02-03T17:24:32.181Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.338077274Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45612024-02-03T17:24:32.181Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.33810323Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45622024-02-03T17:24:32.181Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.338129674Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45632024-02-03T17:24:32.181Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.339126418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45642024-02-03T17:24:32.181Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.339205542Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45652024-02-03T17:24:32.193Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.33981359Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45662024-02-03T17:24:32.193Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.339853934Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45672024-02-03T17:24:32.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.340148436Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45682024-02-03T17:24:32.193Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.340217325Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45692024-02-03T17:24:32.193Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.34024395Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45702024-02-03T17:24:32.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.340265342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45712024-02-03T17:24:32.193Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.340287311Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45722024-02-03T17:24:32.193Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.340309229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45732024-02-03T17:24:32.193Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.340717607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45742024-02-03T17:24:32.193Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341456501Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45752024-02-03T17:24:32.193Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341540537Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45762024-02-03T17:24:32.193Z{"msg":"[0]R flush_numbers[0..12]: [71, 74, 67, 68, 72, 69, 74, 70, 70, 73, 66, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341593753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45772024-02-03T17:24:32.193Z{"msg":"[0]R generation[0..12]: [40, 41, 38, 38, 40, 38, 41, 39, 39, 40, 37, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341619656Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45782024-02-03T17:24:32.193Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341663341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45792024-02-03T17:24:32.193Z{"msg":"[1]R flush_numbers[0..12]: [71, 74, 67, 68, 72, 69, 74, 70, 70, 73, 66, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341688378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45802024-02-03T17:24:32.194Z{"msg":"[1]R generation[0..12]: [40, 41, 38, 38, 40, 38, 41, 39, 39, 40, 37, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341744396Z","Wait for a query_work_queue command to finish before sending IO
45812024-02-03T17:24:32.194Zhostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45822024-02-03T17:24:32.194Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341887376Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45832024-02-03T17:24:32.194Z{"msg":"[2]R flush_numbers[0..12]: [71, 74, 67, 68, 72, 69, 74, 70, 70, 73, 66, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341950408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45842024-02-03T17:24:32.194Z{"msg":"[2]R generation[0..12]: [40, 41, 38, 38, 40, 38, 41, 39, 39, 40, 37, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.341976895Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45852024-02-03T17:24:32.194Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342031969Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45862024-02-03T17:24:32.194Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342119974Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45872024-02-03T17:24:32.194Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342191228Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45882024-02-03T17:24:32.194Z{"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342258985Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45892024-02-03T17:24:32.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342286269Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"":"downstairs","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45902024-02-03T17:24:32.194Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342328683Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45912024-02-03T17:24:32.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342360339Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"0","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45922024-02-03T17:24:32.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342434395Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"1","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45932024-02-03T17:24:32.194Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342463981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"client":"2","":"downstairs","session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45942024-02-03T17:24:32.194Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342533407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45952024-02-03T17:24:32.194Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342562221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45962024-02-03T17:24:32.194Z{"msg":"d724c70b-c21e-4304-b41c-fc6badbfb9ae is now active with session: cc03d09f-6004-43e5-9bbc-fbb107ca5587","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45972024-02-03T17:24:32.194Zcrucible","level":30,"time":"2024-02-03T17:24:31.342717327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45982024-02-03T17:24:32.194Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342793105Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300,"session_id":"cc03d09f-6004-43e5-9bbc-fbb107ca5587"}
45992024-02-03T17:24:32.194Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:31.342863729Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300}
46002024-02-03T17:24:32.194ZLoading write count information from file "/tmp/test_repair_verify.data"
46012024-02-03T17:24:32.194ZRun Repair workload
46022024-02-03T17:24:32.194Z01/30 Write block 499 len 12288 data: 4 5 7
46032024-02-03T17:24:32.195Z02/30 Read block 446 len 8192
46042024-02-03T17:24:32.195Z03/30 Flush
46052024-02-03T17:24:32.195Z04/30 Write block 416 len 20480 data: 4 5 4 4 4
46062024-02-03T17:24:32.195Z05/30 Read block 162 len 8192
46072024-02-03T17:24:32.199Z06/30 Read block 7 len 4096
46082024-02-03T17:24:32.199Z07/30 Write block 485 len 40960 data: 2 3 5 5 4 4 4 4 4 4
46092024-02-03T17:24:32.199Z08/30 Read block 29 len 36864
46102024-02-03T17:24:32.199Z09/30 Write block 138 len 4096 data: 8
46112024-02-03T17:24:32.203Z10/30 Write block 33 len 28672 data: 3 3 4 5 5 5 6
46122024-02-03T17:24:32.203Z11/30 Read block 315 len 32768
46132024-02-03T17:24:32.203Z12/30 Read block 415 len 8192
46142024-02-03T17:24:32.203Z13/30 Read block 156 len 36864
46152024-02-03T17:24:32.207Z14/30 Flush
46162024-02-03T17:24:32.211Z15/30 Write block 196 len 20480 data: 3 2 2 3 4
46172024-02-03T17:24:32.211Z16/30 Read block 403 len 8192
46182024-02-03T17:24:32.211Z17/30 Write block 177 len 20480 data: 3 3 3 3 4
46192024-02-03T17:24:32.214Z18/30 Read block 359 len 24576
46202024-02-03T17:24:32.214Z19/30 Read block 136 len 20480
46212024-02-03T17:24:32.218Z20/30 Write block 537 len 4096 data: 5
46222024-02-03T17:24:32.218Z21/30 Flush
46232024-02-03T17:24:32.222Z22/30 Write block 269 len 36864 data: 4 5 6 6 6 6 8 9 6
46242024-02-03T17:24:32.222Z23/30 Read block 465 len 40960
46252024-02-03T17:24:32.225Z24/30 Read block 377 len 36864
46262024-02-03T17:24:32.225Z25/30 Write block 86 len 16384 data: 4 6 5 6
46272024-02-03T17:24:32.225Z26/30 Read block 246 len 12288
46282024-02-03T17:24:32.230Z27/30 Write block 180 len 16384 data: 4 5 4 3
46292024-02-03T17:24:32.230Z28/30 Write block 150 len 24576 data: 6 6 5 6 7 6
46302024-02-03T17:24:32.230Z29/30 Write block 250 len 4096 data: 7
46312024-02-03T17:24:32.230Z30/30 Read block 248 len 32768
46322024-02-03T17:24:32.235Z----------------------------------------------------------------
46332024-02-03T17:24:32.235Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:27
46342024-02-03T17:24:32.236ZStates: Active Active Active
46352024-02-03T17:24:32.236ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46362024-02-03T17:24:32.236Z 4 Acked 1003 Write 5 Done Done Sent false
46372024-02-03T17:24:32.236Z 5 Acked 1004 Read 2 Done Done Sent false
46382024-02-03T17:24:32.236Z 6 Acked 1005 Read 1 Done Done Sent false
46392024-02-03T17:24:32.236Z 7 Acked 1006 Write 10 Done Done Sent false
46402024-02-03T17:24:32.236Z 8 Acked 1007 Read 9 Done Done Sent false
46412024-02-03T17:24:32.236Z 9 Acked 1008 Write 1 Done Done Sent false
46422024-02-03T17:24:32.236Z 10 Acked 1009 Write 7 Done Done Sent false
46432024-02-03T17:24:32.236Z 11 Acked 1010 Read 8 Done Done Sent false
46442024-02-03T17:24:32.236Z 12 Acked 1011 Read 2 Done Done Sent false
46452024-02-03T17:24:32.237Z 13 Acked 1012 Read 9 Done Done Sent false
46462024-02-03T17:24:32.237Z 14 Acked 1013 Flush 0 Done Done Sent false
46472024-02-03T17:24:32.237Z 15 Acked 1014 Write 5 Done Done Sent false
46482024-02-03T17:24:32.237Z 16 Acked 1015 Read 2 Done Done Sent false
46492024-02-03T17:24:32.237Z 17 Acked 1016 Write 5 Done Done Sent false
46502024-02-03T17:24:32.237Z 18 Acked 1017 Read 6 Done Done Sent false
46512024-02-03T17:24:32.237Z 19 Acked 1018 Read 5 Done Done Sent false
46522024-02-03T17:24:32.237Z 20 Acked 1019 Write 1 Done Done Sent false
46532024-02-03T17:24:32.237Z 21 Acked 1020 Flush 0 Done Done Sent false
46542024-02-03T17:24:32.237Z 22 Acked 1021 Write 9 Done Done Sent false
46552024-02-03T17:24:32.237Z 23 Acked 1022 Read 10 Done Done Sent false
46562024-02-03T17:24:32.237Z 24 Acked 1023 Read 9 Done Done Sent false
46572024-02-03T17:24:32.237Z 25 Acked 1024 Write 4 Done Done Sent false
46582024-02-03T17:24:32.237Z 26 Acked 1025 Read 3 Done Done Sent false
46592024-02-03T17:24:32.237Z 27 Acked 1026 Write 4 Done Done Sent false
46602024-02-03T17:24:32.237Z 28 Acked 1027 Write 6 Done Done Sent false
46612024-02-03T17:24:32.237Z 29 Acked 1028 Write 1 Done Done Sent false
46622024-02-03T17:24:32.237Z 30 Acked 1029 Read 8 Done Done Sent false
46632024-02-03T17:24:32.237Z STATES DS:0 DS:1 DS:2 TOTAL
46642024-02-03T17:24:32.237Z New 0 0 0 0
46652024-02-03T17:24:32.237Z Sent 0 0 27 27
46662024-02-03T17:24:32.238Z Done 27 27 0 54
46672024-02-03T17:24:32.238Z Skipped 0 0 0 0
46682024-02-03T17:24:32.238Z Error 0 0 0 0
46692024-02-03T17:24:32.238ZLast Flush: 1020 1020 1002
46702024-02-03T17:24:32.238ZDownstairs last five completed: 1002 1001 1000
46712024-02-03T17:24:32.238ZUpstairs last five completed: 30 29 28 27 26
46722024-02-03T17:24:32.238Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:31.391168707Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1300}
46732024-02-03T17:24:32.238ZWrote out file "/tmp/test_repair_verify.data"
46742024-02-03T17:24:32.241Z
46752024-02-03T17:24:32.247ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46762024-02-03T17:24:32.274ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46772024-02-03T17:24:32.274Z 0 000-019 40 40 40 71 71 71 F F F
46782024-02-03T17:24:32.278Z 1 020-039 42 42 41 76 76 74 F F F <---
46792024-02-03T17:24:32.278Z 2 040-059 38 38 38 67 67 67 F F F
46802024-02-03T17:24:32.278Z 3 060-079 38 38 38 68 68 68 F F F
46812024-02-03T17:24:32.278Z 4 080-099 40 40 40 72 72 72 T T F <---
46822024-02-03T17:24:32.278Z 5 100-119 38 38 38 69 69 69 F F F
46832024-02-03T17:24:32.278Z 6 120-139 42 42 41 76 76 74 F F F <---
46842024-02-03T17:24:32.278Z 7 140-159 39 39 39 70 70 70 T T F <---
46852024-02-03T17:24:32.278Z 8 160-179 42 42 39 77 77 70 F F F <---
46862024-02-03T17:24:32.278Z 9 180-199 42 42 40 77 77 73 T T F <---
46872024-02-03T17:24:32.278Z 10 200-219 42 42 37 77 77 66 F F F <---
46882024-02-03T17:24:32.279Z 11 220-239 40 40 40 73 73 73 F F F
46892024-02-03T17:24:32.279Z 12 240-259 40 40 40 73 73 73 T T F <---
46902024-02-03T17:24:32.279Z 13 260-279 30 30 30 54 54 54 T T F <---
46912024-02-03T17:24:32.279Z 14 280-299 41 41 41 74 74 74 F F F
46922024-02-03T17:24:32.279Z 15 300-319 38 38 38 67 67 67 F F F
46932024-02-03T17:24:32.279Z 16 320-339 36 36 36 65 65 65 F F F
46942024-02-03T17:24:32.279Z 17 340-359 40 40 40 71 71 71 F F F
46952024-02-03T17:24:32.279Z 18 360-379 39 39 39 70 70 70 F F F
46962024-02-03T17:24:32.279Z 19 380-399 34 34 34 62 62 62 F F F
46972024-02-03T17:24:32.279Z 20 400-419 42 42 34 76 76 62 F F F <---
46982024-02-03T17:24:32.279Z 21 420-439 42 42 34 76 76 62 F F F <---
46992024-02-03T17:24:32.279Z 22 440-459 38 38 38 67 67 67 F F F
47002024-02-03T17:24:32.279Z 23 460-479 39 39 39 70 70 70 F F F
47012024-02-03T17:24:32.280Z 24 480-499 42 42 42 76 76 75 F F F <---
47022024-02-03T17:24:32.280Z 25 500-519 42 42 42 75 75 75 F F F
47032024-02-03T17:24:32.280Z 26 520-539 42 42 36 77 77 64 F F F <---
47042024-02-03T17:24:32.280Z 27 540-559 36 36 36 64 64 64 F F F
47052024-02-03T17:24:32.280Z 28 560-579 40 40 40 71 71 71 F F F
47062024-02-03T17:24:32.280Z 29 580-599 40 40 40 71 71 71 F F F
47072024-02-03T17:24:32.280ZMax gen: 42, Max flush: 77
47082024-02-03T17:24:32.280ZError: Difference in extent metadata found!
47092024-02-03T17:24:32.284ZA Difference in extent metadata is expected here
47102024-02-03T17:24:32.284ZOn loop 21
47112024-02-03T17:24:32.284Z
47122024-02-03T17:24:32.284ZVerifying data now
47132024-02-03T17:24:37.440ZNo change after verify
47142024-02-03T17:24:37.440ZLoop: 21 Downstairs dump after verify (and repair):
47152024-02-03T17:24:37.465ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47162024-02-03T17:24:37.465Z 0 000-019 40 40 40 71 71 71 F F F
47172024-02-03T17:24:37.465Z 1 020-039 42 42 42 76 76 76 F F F
47182024-02-03T17:24:37.465Z 2 040-059 38 38 38 67 67 67 F F F
47192024-02-03T17:24:37.465Z 3 060-079 38 38 38 68 68 68 F F F
47202024-02-03T17:24:37.469Z 4 080-099 43 43 43 78 78 78 F F F
47212024-02-03T17:24:37.469Z 5 100-119 38 38 38 69 69 69 F F F
47222024-02-03T17:24:37.469Z 6 120-139 42 42 42 76 76 76 F F F
47232024-02-03T17:24:37.469Z 7 140-159 43 43 43 78 78 78 F F F
47242024-02-03T17:24:37.469Z 8 160-179 42 42 42 77 77 77 F F F
47252024-02-03T17:24:37.469Z 9 180-199 43 43 43 78 78 78 F F F
47262024-02-03T17:24:37.469Z 10 200-219 42 42 42 77 77 77 F F F
47272024-02-03T17:24:37.469Z 11 220-239 40 40 40 73 73 73 F F F
47282024-02-03T17:24:37.469Z 12 240-259 43 43 43 78 78 78 F F F
47292024-02-03T17:24:37.469Z 13 260-279 43 43 43 78 78 78 F F F
47302024-02-03T17:24:37.469Z 14 280-299 41 41 41 74 74 74 F F F
47312024-02-03T17:24:37.469Z 15 300-319 38 38 38 67 67 67 F F F
47322024-02-03T17:24:37.469Z 16 320-339 36 36 36 65 65 65 F F F
47332024-02-03T17:24:37.469Z 17 340-359 40 40 40 71 71 71 F F F
47342024-02-03T17:24:37.469Z 18 360-379 39 39 39 70 70 70 F F F
47352024-02-03T17:24:37.469Z 19 380-399 34 34 34 62 62 62 F F F
47362024-02-03T17:24:37.469Z 20 400-419 42 42 42 76 76 76 F F F
47372024-02-03T17:24:37.469Z 21 420-439 42 42 42 76 76 76 F F F
47382024-02-03T17:24:37.469Z 22 440-459 38 38 38 67 67 67 F F F
47392024-02-03T17:24:37.470Z 23 460-479 39 39 39 70 70 70 F F F
47402024-02-03T17:24:37.470Z 24 480-499 42 42 42 76 76 76 F F F
47412024-02-03T17:24:37.470Z 25 500-519 42 42 42 75 75 75 F F F
47422024-02-03T17:24:37.470Z 26 520-539 42 42 42 77 77 77 F F F
47432024-02-03T17:24:37.470Z 27 540-559 36 36 36 64 64 64 F F F
47442024-02-03T17:24:37.470Z 28 560-579 40 40 40 71 71 71 F F F
47452024-02-03T17:24:37.470Z 29 580-599 40 40 40 71 71 71 F F F
47462024-02-03T17:24:37.470ZMax gen: 43, Max flush: 78
47472024-02-03T17:24:37.473Z
47482024-02-03T17:24:37.473ZBegin loop 22
47492024-02-03T17:24:37.473ZDownstairs to restart: 1
47502024-02-03T17:24:37.505Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.664601427Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308}
47512024-02-03T17:24:37.505Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665147175Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308}
47522024-02-03T17:24:37.505Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665210092Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308}
47532024-02-03T17:24:37.509Z{"msg":"Crucible b442f1fc-0831-487b-8c86-ee75ed699411 has session id: 48dd70df-86d9-4b55-a9a6-b6459c2471ff","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665246059Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47542024-02-03T17:24:37.509Z{"msg":"Upstairs opts: Upstairs UUID: b442f1fc-0831-487b-8c86-ee75ed699411, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
47552024-02-03T17:24:37.509Z,"time":"2024-02-03T17:24:36.665306846Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47562024-02-03T17:24:37.509Z{"msg":"Crucible stats registered with UUID: b442f1fc-0831-487b-8c86-ee75ed699411","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665359312Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47572024-02-03T17:24:37.509Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.66544869Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308}
47582024-02-03T17:24:37.509Z{"msg":"b442f1fc-0831-487b-8c86-ee75ed699411 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665697787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47592024-02-03T17:24:37.509Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665754882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47602024-02-03T17:24:37.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665781862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47612024-02-03T17:24:37.509Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665803852Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47622024-02-03T17:24:37.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665826575Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47632024-02-03T17:24:37.509Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665850313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47642024-02-03T17:24:37.509Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665881986Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47652024-02-03T17:24:37.509Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665919334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"io task","client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47662024-02-03T17:24:37.509Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.665945278Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"io task","client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47672024-02-03T17:24:37.509Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.66607752Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"io task","client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47682024-02-03T17:24:37.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.666360692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"io task","client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47692024-02-03T17:24:37.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.666417944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"io task","client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47702024-02-03T17:24:37.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.666559821Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"io task","client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47712024-02-03T17:24:37.509Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.66785929Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47722024-02-03T17:24:37.509Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.6679241Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47732024-02-03T17:24:37.509Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.667971253Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47742024-02-03T17:24:37.509Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.668001871Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47752024-02-03T17:24:37.510Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.668790067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47762024-02-03T17:24:37.510Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.668906308Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47772024-02-03T17:24:37.514Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.668947071Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47782024-02-03T17:24:37.514Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.669237098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47792024-02-03T17:24:37.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.670083763Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47802024-02-03T17:24:37.514Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.670155575Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47812024-02-03T17:24:37.514Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.670182445Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47822024-02-03T17:24:37.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.670276388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47832024-02-03T17:24:37.514Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.670328198Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47842024-02-03T17:24:37.514Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.670353695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47852024-02-03T17:24:37.514Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.670375396Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47862024-02-03T17:24:37.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671188612Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47872024-02-03T17:24:37.514Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671301571Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47882024-02-03T17:24:37.514Z{"msg":"[0]R flush_numbers[0..12]: [71, 76, 67, 68, 78, 69, 76, 78, 77, 78, 77, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.67132927Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47892024-02-03T17:24:37.514Z{"msg":"[0]R generation[0..12]: [40, 42, 38, 38, 43, 38, 42, 43, 42, 43, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671372361Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47902024-02-03T17:24:37.514Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671398686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47912024-02-03T17:24:37.514Z{"msg":"[1]R flush_numbers[0..12]: [71, 76, 67, 68, 78, 69, 76, 78, 77, 78, 77, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671441275Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47922024-02-03T17:24:37.514Z{"msg":"[1]R generation[0..12]: [40, 42, 38, 38, 43, 38, 42, 43, 42, 43, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671466457Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47932024-02-03T17:24:37.514Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671509484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47942024-02-03T17:24:37.515ZWait for a query_work_queue command to finish before sending IO
47952024-02-03T17:24:37.515Z{"msg":"[2]R flush_numbers[0..12]: [71, 76, 67, 68, 78, 69, 76, 78, 77, 78, 77, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671541524Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47962024-02-03T17:24:37.515Z{"msg":"[2]R generation[0..12]: [40, 42, 38, 38, 43, 38, 42, 43, 42, 43, 42, 40]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671590538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47972024-02-03T17:24:37.515Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671616351Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47982024-02-03T17:24:37.515Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671638615Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
47992024-02-03T17:24:37.515Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671662072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48002024-02-03T17:24:37.515Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671702655Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48012024-02-03T17:24:37.515Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671726574Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"":"downstairs","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48022024-02-03T17:24:37.515Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671767714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48032024-02-03T17:24:37.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671789826Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"0","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48042024-02-03T17:24:37.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671830526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"1","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48052024-02-03T17:24:37.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671854415Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"client":"2","":"downstairs","session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48062024-02-03T17:24:37.515Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671904081Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48072024-02-03T17:24:37.515Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671926594Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48082024-02-03T17:24:37.515Z{"msg":"b442f1fc-0831-487b-8c86-ee75ed699411 is now active with session: 48dd70df-86d9-4b55-a9a6-b6459c2471ff","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671949407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48092024-02-03T17:24:37.515Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.671969062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308,"session_id":"48dd70df-86d9-4b55-a9a6-b6459c2471ff"}
48102024-02-03T17:24:37.515Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:36.672017349Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308}
48112024-02-03T17:24:37.515ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48122024-02-03T17:24:37.515ZLoading write count information from file "/tmp/test_repair_verify.data"
48132024-02-03T17:24:37.515ZRun Repair workload
48142024-02-03T17:24:37.515Z01/30 Write block 434 len 32768 data: 2 2 2 2 2 3 5 5
48152024-02-03T17:24:37.519Z02/30 Read block 12 len 32768
48162024-02-03T17:24:37.519Z03/30 Read block 234 len 28672
48172024-02-03T17:24:37.522Z04/30 Read block 459 len 12288
48182024-02-03T17:24:37.522Z05/30 Write block 271 len 36864 data: 7 7 7 7 9 10 7 5 3
48192024-02-03T17:24:37.522Z06/30 Flush
48202024-02-03T17:24:37.526Z07/30 Write block 169 len 12288 data: 6 6 5
48212024-02-03T17:24:37.531Z08/30 Write block 104 len 8192 data: 3 4
48222024-02-03T17:24:37.531Z09/30 Flush
48232024-02-03T17:24:37.535Z10/30 Flush
48242024-02-03T17:24:37.535Z11/30 Write block 136 len 28672 data: 8 8 9 7 5 5 4
48252024-02-03T17:24:37.535Z12/30 Write block 489 len 40960 data: 5 5 5 5 5 5 4 3 4 4
48262024-02-03T17:24:37.535Z13/30 Write block 231 len 16384 data: 3 3 3 4
48272024-02-03T17:24:37.538Z14/30 Read block 579 len 8192
48282024-02-03T17:24:37.538Z15/30 Write block 250 len 28672 data: 8 5 3 3 5 6 5
48292024-02-03T17:24:37.538Z16/30 Write block 333 len 8192 data: 6 5
48302024-02-03T17:24:37.541Z17/30 Read block 70 len 36864
48312024-02-03T17:24:37.541Z18/30 Read block 547 len 36864
48322024-02-03T17:24:37.544Z19/30 Read block 495 len 40960
48332024-02-03T17:24:37.544Z20/30 Read block 517 len 20480
48342024-02-03T17:24:37.547Z21/30 Read block 351 len 32768
48352024-02-03T17:24:37.547Z22/30 Read block 187 len 8192
48362024-02-03T17:24:37.550Z23/30 Read block 273 len 32768
48372024-02-03T17:24:37.553Z24/30 Write block 105 len 40960 data: 5 4 5 5 5 5 6 6 6 7
48382024-02-03T17:24:37.553Z25/30 Write block 10 len 36864 data: 5 5 5 5 4 3 3 2 2
48392024-02-03T17:24:37.553Z26/30 Read block 84 len 12288
48402024-02-03T17:24:37.556Z27/30 Write block 555 len 24576 data: 5 6 5 5 5 4
48412024-02-03T17:24:37.559Z28/30 Write block 445 len 32768 data: 5 5 5 6 6 6 5 5
48422024-02-03T17:24:37.559Z29/30 Write block 431 len 20480 data: 3 3 2 3 3
48432024-02-03T17:24:37.559Z30/30 Read block 105 len 8192
48442024-02-03T17:24:37.563Z----------------------------------------------------------------
48452024-02-03T17:24:37.563Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48462024-02-03T17:24:37.563ZStates: Active Active Active
48472024-02-03T17:24:37.563ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48482024-02-03T17:24:37.563Z 1 Acked 1000 Write 8 Done Sent Done false
48492024-02-03T17:24:37.563Z 2 Acked 1001 Read 8 Done Sent Done false
48502024-02-03T17:24:37.563Z 3 Acked 1002 Read 7 Done Sent Done false
48512024-02-03T17:24:37.563Z 4 Acked 1003 Read 3 Done Sent Done false
48522024-02-03T17:24:37.563Z 5 Acked 1004 Write 9 Done Sent Done false
48532024-02-03T17:24:37.563Z 6 Acked 1005 Flush 0 Done Sent Done false
48542024-02-03T17:24:37.563Z 7 Acked 1006 Write 3 Done Sent Done false
48552024-02-03T17:24:37.563Z 8 Acked 1007 Write 2 Done Sent Done false
48562024-02-03T17:24:37.563Z 9 Acked 1008 Flush 0 Done Sent Done false
48572024-02-03T17:24:37.563Z 10 Acked 1009 Flush 0 Done Sent Done false
48582024-02-03T17:24:37.563Z 11 Acked 1010 Write 7 Done Sent Done false
48592024-02-03T17:24:37.563Z 12 Acked 1011 Write 10 Done Sent Done false
48602024-02-03T17:24:37.563Z 13 Acked 1012 Write 4 Done Sent Done false
48612024-02-03T17:24:37.563Z 14 Acked 1013 Read 2 Done Sent Done false
48622024-02-03T17:24:37.563Z 15 Acked 1014 Write 7 Done Sent Done false
48632024-02-03T17:24:37.563Z 16 Acked 1015 Write 2 Done Sent Done false
48642024-02-03T17:24:37.563Z 17 Acked 1016 Read 9 Done Sent Done false
48652024-02-03T17:24:37.563Z 18 Acked 1017 Read 9 Done Sent Done false
48662024-02-03T17:24:37.563Z 19 Acked 1018 Read 10 Done Sent Done false
48672024-02-03T17:24:37.564Z 20 Acked 1019 Read 5 Done Sent Done false
48682024-02-03T17:24:37.564Z 21 Acked 1020 Read 8 Done Sent Done false
48692024-02-03T17:24:37.564Z 22 Acked 1021 Read 2 Done Sent Done false
48702024-02-03T17:24:37.564Z 23 Acked 1022 Read 8 Done Sent Done false
48712024-02-03T17:24:37.564Z 24 Acked 1023 Write 10 Done Sent Done false
48722024-02-03T17:24:37.564Z 25 Acked 1024 Write 9 Done Sent Done false
48732024-02-03T17:24:37.564Z 26 Acked 1025 Read 3 Done Sent Done false
48742024-02-03T17:24:37.564Z 27 Acked 1026 Write 6 Done Sent Done false
48752024-02-03T17:24:37.564Z 28 Acked 1027 Write 8 Done Sent Done false
48762024-02-03T17:24:37.564Z 29 Acked 1028 Write 5 Done Sent Done false
48772024-02-03T17:24:37.564Z 30 Acked 1029 Read 2 Done Sent Done false
48782024-02-03T17:24:37.564Z STATES DS:0 DS:1 DS:2 TOTAL
48792024-02-03T17:24:37.564Z New 0 0 0 0
48802024-02-03T17:24:37.564Z Sent 0 30 0 30
48812024-02-03T17:24:37.564Z Done 30 0 30 60
48822024-02-03T17:24:37.564Z Skipped 0 0 0 0
48832024-02-03T17:24:37.564Z Error 0 0 0 0
48842024-02-03T17:24:37.564ZLast Flush: 1009 0 1009
48852024-02-03T17:24:37.564ZDownstairs last five completed:
48862024-02-03T17:24:37.564ZUpstairs last five completed: 30 29 28 27 26
48872024-02-03T17:24:37.564Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:36.721387316Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1308}
48882024-02-03T17:24:37.564ZWrote out file "/tmp/test_repair_verify.data"
48892024-02-03T17:24:37.569Z
48902024-02-03T17:24:37.576ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48912024-02-03T17:24:37.608ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48922024-02-03T17:24:37.608Z 0 000-019 40 40 40 71 71 71 T F T <---
48932024-02-03T17:24:37.608Z 1 020-039 42 42 42 76 76 76 F F F
48942024-02-03T17:24:37.608Z 2 040-059 38 38 38 67 67 67 F F F
48952024-02-03T17:24:37.608Z 3 060-079 38 38 38 68 68 68 F F F
48962024-02-03T17:24:37.608Z 4 080-099 43 43 43 78 78 78 F F F
48972024-02-03T17:24:37.608Z 5 100-119 44 38 44 80 69 80 T F T <---
48982024-02-03T17:24:37.611Z 6 120-139 42 42 42 76 76 76 T F T <---
48992024-02-03T17:24:37.611Z 7 140-159 43 43 43 78 78 78 T F T <---
49002024-02-03T17:24:37.611Z 8 160-179 44 42 44 80 77 80 F F F <---
49012024-02-03T17:24:37.611Z 9 180-199 43 43 43 78 78 78 F F F
49022024-02-03T17:24:37.611Z 10 200-219 42 42 42 77 77 77 F F F
49032024-02-03T17:24:37.611Z 11 220-239 40 40 40 73 73 73 T F T <---
49042024-02-03T17:24:37.612Z 12 240-259 43 43 43 78 78 78 T F T <---
49052024-02-03T17:24:37.612Z 13 260-279 44 43 44 79 78 79 F F F <---
49062024-02-03T17:24:37.612Z 14 280-299 41 41 41 74 74 74 F F F
49072024-02-03T17:24:37.612Z 15 300-319 38 38 38 67 67 67 F F F
49082024-02-03T17:24:37.612Z 16 320-339 36 36 36 65 65 65 T F T <---
49092024-02-03T17:24:37.612Z 17 340-359 40 40 40 71 71 71 F F F
49102024-02-03T17:24:37.612Z 18 360-379 39 39 39 70 70 70 F F F
49112024-02-03T17:24:37.612Z 19 380-399 34 34 34 62 62 62 F F F
49122024-02-03T17:24:37.612Z 20 400-419 42 42 42 76 76 76 F F F
49132024-02-03T17:24:37.612Z 21 420-439 44 42 44 79 76 79 T F T <---
49142024-02-03T17:24:37.612Z 22 440-459 44 38 44 79 67 79 T F T <---
49152024-02-03T17:24:37.612Z 23 460-479 39 39 39 70 70 70 F F F
49162024-02-03T17:24:37.612Z 24 480-499 42 42 42 76 76 76 T F T <---
49172024-02-03T17:24:37.612Z 25 500-519 42 42 42 75 75 75 F F F
49182024-02-03T17:24:37.612Z 26 520-539 42 42 42 77 77 77 F F F
49192024-02-03T17:24:37.612Z 27 540-559 36 36 36 64 64 64 T F T <---
49202024-02-03T17:24:37.612Z 28 560-579 40 40 40 71 71 71 T F T <---
49212024-02-03T17:24:37.612Z 29 580-599 40 40 40 71 71 71 F F F
49222024-02-03T17:24:37.612ZMax gen: 44, Max flush: 80
49232024-02-03T17:24:37.612ZError: Difference in extent metadata found!
49242024-02-03T17:24:37.615ZA Difference in extent metadata is expected here
49252024-02-03T17:24:37.616ZOn loop 22
49262024-02-03T17:24:37.616Z
49272024-02-03T17:24:37.619ZVerifying data now
49282024-02-03T17:24:42.856ZNo change after verify
49292024-02-03T17:24:42.856ZLoop: 22 Downstairs dump after verify (and repair):
49302024-02-03T17:24:42.881ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49312024-02-03T17:24:42.881Z 0 000-019 45 45 45 81 81 81 F F F
49322024-02-03T17:24:42.881Z 1 020-039 42 42 42 76 76 76 F F F
49332024-02-03T17:24:42.881Z 2 040-059 38 38 38 67 67 67 F F F
49342024-02-03T17:24:42.881Z 3 060-079 38 38 38 68 68 68 F F F
49352024-02-03T17:24:42.881Z 4 080-099 43 43 43 78 78 78 F F F
49362024-02-03T17:24:42.881Z 5 100-119 45 45 45 81 81 81 F F F
49372024-02-03T17:24:42.881Z 6 120-139 45 45 45 81 81 81 F F F
49382024-02-03T17:24:42.881Z 7 140-159 45 45 45 81 81 81 F F F
49392024-02-03T17:24:42.881Z 8 160-179 44 44 44 80 80 80 F F F
49402024-02-03T17:24:42.884Z 9 180-199 43 43 43 78 78 78 F F F
49412024-02-03T17:24:42.884Z 10 200-219 42 42 42 77 77 77 F F F
49422024-02-03T17:24:42.884Z 11 220-239 45 45 45 81 81 81 F F F
49432024-02-03T17:24:42.884Z 12 240-259 45 45 45 81 81 81 F F F
49442024-02-03T17:24:42.884Z 13 260-279 44 44 44 79 79 79 F F F
49452024-02-03T17:24:42.885Z 14 280-299 41 41 41 74 74 74 F F F
49462024-02-03T17:24:42.885Z 15 300-319 38 38 38 67 67 67 F F F
49472024-02-03T17:24:42.885Z 16 320-339 45 45 45 81 81 81 F F F
49482024-02-03T17:24:42.885Z 17 340-359 40 40 40 71 71 71 F F F
49492024-02-03T17:24:42.885Z 18 360-379 39 39 39 70 70 70 F F F
49502024-02-03T17:24:42.885Z 19 380-399 34 34 34 62 62 62 F F F
49512024-02-03T17:24:42.885Z 20 400-419 42 42 42 76 76 76 F F F
49522024-02-03T17:24:42.885Z 21 420-439 45 45 45 81 81 81 F F F
49532024-02-03T17:24:42.885Z 22 440-459 45 45 45 81 81 81 F F F
49542024-02-03T17:24:42.885Z 23 460-479 39 39 39 70 70 70 F F F
49552024-02-03T17:24:42.885Z 24 480-499 45 45 45 81 81 81 F F F
49562024-02-03T17:24:42.885Z 25 500-519 42 42 42 75 75 75 F F F
49572024-02-03T17:24:42.885Z 26 520-539 42 42 42 77 77 77 F F F
49582024-02-03T17:24:42.885Z 27 540-559 45 45 45 81 81 81 F F F
49592024-02-03T17:24:42.885Z 28 560-579 45 45 45 81 81 81 F F F
49602024-02-03T17:24:42.885Z 29 580-599 40 40 40 71 71 71 F F F
49612024-02-03T17:24:42.885ZMax gen: 45, Max flush: 81
49622024-02-03T17:24:42.889Z
49632024-02-03T17:24:42.889ZBegin loop 23
49642024-02-03T17:24:42.889ZDownstairs to restart: 0
49652024-02-03T17:24:42.919Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.078938373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316}
49662024-02-03T17:24:42.919ZCrucible runtime is spawned
49672024-02-03T17:24:42.924Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079578894Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316}
49682024-02-03T17:24:42.924Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079656378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316}
49692024-02-03T17:24:42.924Z{"msg":"Crucible d1b5f3c2-d425-42b2-8b6f-4b8caf330c88 has session id: 06be20fa-17a8-48dd-910d-a97ddc7f6d8a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079716872Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49702024-02-03T17:24:42.924Z{"msg":"Upstairs opts: Upstairs UUID: d1b5f3c2-d425-42b2-8b6f-4b8caf330c88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079808958Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49712024-02-03T17:24:42.924Z{"msg":"Crucible stats registered with UUID: d1b5f3c2-d425-42b2-8b6f-4b8caf330c88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079842063Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49722024-02-03T17:24:42.924Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079886142Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316}
49732024-02-03T17:24:42.924Z{"msg":"d1b5f3c2-d425-42b2-8b6f-4b8caf330c88 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079907642Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49742024-02-03T17:24:42.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079947224Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49752024-02-03T17:24:42.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.079972866Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49762024-02-03T17:24:42.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080027958Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49772024-02-03T17:24:42.925Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080132525Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49782024-02-03T17:24:42.925Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080210686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49792024-02-03T17:24:42.925Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080313853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49802024-02-03T17:24:42.925Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080355121Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49812024-02-03T17:24:42.925Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080395793Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49822024-02-03T17:24:42.925Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080435164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49832024-02-03T17:24:42.925Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.08047513Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"io task","client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49842024-02-03T17:24:42.925Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080745135Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"io task","client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49852024-02-03T17:24:42.925Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.080837311Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"io task","client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49862024-02-03T17:24:42.925Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.081961822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49872024-02-03T17:24:42.925Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.082172439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49882024-02-03T17:24:42.925Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.082216252Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49892024-02-03T17:24:42.925Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.082289906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49902024-02-03T17:24:42.926Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.083067065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49912024-02-03T17:24:42.926Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.083245347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49922024-02-03T17:24:42.926Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.083298229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49932024-02-03T17:24:42.926Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.083476591Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49942024-02-03T17:24:42.931Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.084186604Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49952024-02-03T17:24:42.931Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.08425865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49962024-02-03T17:24:42.931Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.084304426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49972024-02-03T17:24:42.931Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.084368598Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49982024-02-03T17:24:42.931Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.08441368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
49992024-02-03T17:24:42.931Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.084475489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50002024-02-03T17:24:42.931Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.084622652Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50012024-02-03T17:24:42.931Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.085327023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50022024-02-03T17:24:42.931Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.085415229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50032024-02-03T17:24:42.931Z{"msg":"[0]R flush_numbers[0..12]: [81, 76, 67, 68, 78, 81, 81, 81, 80, 78, 77, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.085491908Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50042024-02-03T17:24:42.932Z{"msg":"[0]R generation[0..12]: [45, 42, 38, 38, 43, 45, 45, 45, 44, 43, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.085566178Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50052024-02-03T17:24:42.932Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.085638791Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50062024-02-03T17:24:42.932Z{"msg":"[1]R flush_numbers[0..12]: [81, 76, 67, 68, 78, 81, 81, 81, 80, 78, 77, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.085715383Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50072024-02-03T17:24:42.932Z{"msg":"[1]R generation[0..12]: [45, 42, 38, 38, 43, 45, 45, 45, 44, 43, 42, 45]","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO
50082024-02-03T17:24:42.932Z","level":30,"time":"2024-02-03T17:24:42.085883686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50092024-02-03T17:24:42.932Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.085959327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50102024-02-03T17:24:42.932Z{"msg":"[2]R flush_numbers[0..12]: [81, 76, 67, 68, 78, 81, 81, 81, 80, 78, 77, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086035208Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50112024-02-03T17:24:42.932Z{"msg":"[2]R generation[0..12]: [45, 42, 38, 38, 43, 45, 45, 45, 44, 43, 42, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086107927Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50122024-02-03T17:24:42.932Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086155342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50132024-02-03T17:24:42.932Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086270408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50142024-02-03T17:24:42.932Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086319944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50152024-02-03T17:24:42.932Z{"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.08636229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50162024-02-03T17:24:42.932Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086433514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"":"downstairs","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50172024-02-03T17:24:42.933Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086507515Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50182024-02-03T17:24:42.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086574639Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"0","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50192024-02-03T17:24:42.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086646472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"1","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50202024-02-03T17:24:42.933Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50212024-02-03T17:24:42.933Z,"time":"2024-02-03T17:24:42.086721074Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"client":"2","":"downstairs","session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50222024-02-03T17:24:42.933Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086793186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50232024-02-03T17:24:42.933Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086844079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50242024-02-03T17:24:42.933Z{"msg":"d1b5f3c2-d425-42b2-8b6f-4b8caf330c88 is now active with session: 06be20fa-17a8-48dd-910d-a97ddc7f6d8a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086868682Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50252024-02-03T17:24:42.933Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086888475Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316,"session_id":"06be20fa-17a8-48dd-910d-a97ddc7f6d8a"}
50262024-02-03T17:24:42.933Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:42.086927999Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316}
50272024-02-03T17:24:42.933ZLoading write count information from file "/tmp/test_repair_verify.data"
50282024-02-03T17:24:42.933ZRun Repair workload
50292024-02-03T17:24:42.933Z01/30 Write block 245 len 32768 data: 2 4 5 5 6 9 6 4
50302024-02-03T17:24:42.934Z02/30 Read block 590 len 12288
50312024-02-03T17:24:42.937Z03/30 Flush
50322024-02-03T17:24:42.937Z04/30 Write block 101 len 20480 data: 2 3 3 4 6
50332024-02-03T17:24:42.937Z05/30 Write block 99 len 40960 data: 3 3 3 4 4 5 7 5 6 6
50342024-02-03T17:24:42.938Z06/30 Read block 158 len 12288
50352024-02-03T17:24:42.941Z07/30 Read block 448 len 36864
50362024-02-03T17:24:42.944Z08/30 Write block 199 len 32768 data: 4 5 3 3 3 2 3 3
50372024-02-03T17:24:42.944Z09/30 Read block 380 len 32768
50382024-02-03T17:24:42.948Z10/30 Flush
50392024-02-03T17:24:42.951Z11/30 Write block 250 len 4096 data: 10
50402024-02-03T17:24:42.951Z12/30 Read block 540 len 4096
50412024-02-03T17:24:42.951Z13/30 Read block 54 len 28672
50422024-02-03T17:24:42.951Z14/30 Read block 422 len 12288
50432024-02-03T17:24:42.956Z15/30 Write block 575 len 28672 data: 4 4 5 6 7 7 6
50442024-02-03T17:24:42.956Z16/30 Read block 111 len 8192
50452024-02-03T17:24:42.959Z17/30 Write block 148 len 16384 data: 3 4 7 7
50462024-02-03T17:24:42.959Z18/30 Read block 145 len 40960
50472024-02-03T17:24:42.960Z19/30 Write block 150 len 8192 data: 8 8
50482024-02-03T17:24:42.960Z20/30 Write block 322 len 20480 data: 4 4 4 5 5
50492024-02-03T17:24:42.960Z21/30 Read block 356 len 24576
50502024-02-03T17:24:42.963Z22/30 Write block 481 len 36864 data: 4 4 3 2 3 4 6 6 6
50512024-02-03T17:24:42.963Z23/30 Read block 498 len 20480
50522024-02-03T17:24:42.966Z24/30 Read block 104 len 8192
50532024-02-03T17:24:42.966Z25/30 Flush
50542024-02-03T17:24:42.970Z26/30 Write block 4 len 4096 data: 5
50552024-02-03T17:24:42.970Z27/30 Read block 313 len 16384
50562024-02-03T17:24:42.974Z28/30 Read block 313 len 24576
50572024-02-03T17:24:42.974Z29/30 Write block 284 len 24576 data: 4 3 3 5 5 5
50582024-02-03T17:24:42.980Z30/30 Write block 254 len 32768 data: 6 7 6 3 3 3 5 5
50592024-02-03T17:24:42.980Z----------------------------------------------------------------
50602024-02-03T17:24:42.980Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50612024-02-03T17:24:42.980ZStates: Active Active Active
50622024-02-03T17:24:42.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50632024-02-03T17:24:42.980Z 1 Acked 1000 Write 8 Done Done Done false
50642024-02-03T17:24:42.980Z 2 Acked 1001 Read 3 Done Done Done false
50652024-02-03T17:24:42.981Z 3 Acked 1002 Flush 0 Sent Done Done false
50662024-02-03T17:24:42.981Z 4 Acked 1003 Write 5 Sent Done Done false
50672024-02-03T17:24:42.981Z 5 Acked 1004 Write 10 Sent Done Done false
50682024-02-03T17:24:42.981Z 6 Acked 1005 Read 3 Sent Done Done false
50692024-02-03T17:24:42.981Z 7 Acked 1006 Read 9 Sent Done Done false
50702024-02-03T17:24:42.981Z 8 Acked 1007 Write 8 Sent Done Done false
50712024-02-03T17:24:42.981Z 9 Acked 1008 Read 8 Sent Done Done false
50722024-02-03T17:24:42.981Z 10 Acked 1009 Flush 0 Sent Done Done false
50732024-02-03T17:24:42.981Z 11 Acked 1010 Write 1 Sent Done Done false
50742024-02-03T17:24:42.981Z 12 Acked 1011 Read 1 Sent Done Done false
50752024-02-03T17:24:42.981Z 13 Acked 1012 Read 7 Sent Done Done false
50762024-02-03T17:24:42.981Z 14 Acked 1013 Read 3 Sent Done Done false
50772024-02-03T17:24:42.982Z 15 Acked 1014 Write 7 Sent Done Done false
50782024-02-03T17:24:42.982Z 16 Acked 1015 Read 2 Sent Done Done false
50792024-02-03T17:24:42.982Z 17 Acked 1016 Write 4 Sent Done Done false
50802024-02-03T17:24:42.982Z 18 Acked 1017 Read 10 Sent Done Done false
50812024-02-03T17:24:42.982Z 19 Acked 1018 Write 2 Sent Done Done false
50822024-02-03T17:24:42.982Z 20 Acked 1019 Write 5 Sent Done Done false
50832024-02-03T17:24:42.982Z 21 Acked 1020 Read 6 Sent Done Done false
50842024-02-03T17:24:42.982Z 22 Acked 1021 Write 9 Sent Done Done false
50852024-02-03T17:24:42.982Z 23 Acked 1022 Read 5 Sent Done Done false
50862024-02-03T17:24:42.982Z 24 Acked 1023 Read 2 Sent Done Done false
50872024-02-03T17:24:42.982Z 25 Acked 1024 Flush 0 Sent Done Done false
50882024-02-03T17:24:42.982Z 26 Acked 1025 Write 1 Sent Done Done false
50892024-02-03T17:24:42.982Z 27 Acked 1026 Read 4 Sent Done Done false
50902024-02-03T17:24:42.983Z 28 Acked 1027 Read 6 Sent Done Done false
50912024-02-03T17:24:42.983Z 29 Acked 1028 Write 6 Sent Done Done false
50922024-02-03T17:24:42.983Z 30 Acked 1029 Write 8 Sent Sent Sent false
50932024-02-03T17:24:42.983Z STATES DS:0 DS:1 DS:2 TOTAL
50942024-02-03T17:24:42.983Z New 0 0 0 0
50952024-02-03T17:24:42.983Z Sent 28 1 1 30
50962024-02-03T17:24:42.983Z Done 2 29 29 60
50972024-02-03T17:24:42.983Z Skipped 0 0 0 0
50982024-02-03T17:24:42.983Z Error 0 0 0 0
50992024-02-03T17:24:42.983ZLast Flush: 0 1024 1024
51002024-02-03T17:24:42.983ZDownstairs last five completed:
51012024-02-03T17:24:42.984ZUpstairs last five completed: 30 29 28 27 26
51022024-02-03T17:24:42.984Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:42.136553718Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1316}
51032024-02-03T17:24:42.984ZWrote out file "/tmp/test_repair_verify.data"
51042024-02-03T17:24:42.989Z
51052024-02-03T17:24:43.011ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51062024-02-03T17:24:43.018ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51072024-02-03T17:24:43.018Z 0 000-019 45 45 45 81 81 81 F T T <---
51082024-02-03T17:24:43.018Z 1 020-039 42 42 42 76 76 76 F F F
51092024-02-03T17:24:43.018Z 2 040-059 38 38 38 67 67 67 F F F
51102024-02-03T17:24:43.018Z 3 060-079 38 38 38 68 68 68 F F F
51112024-02-03T17:24:43.018Z 4 080-099 43 46 46 78 83 83 F F F <---
51122024-02-03T17:24:43.024Z 5 100-119 45 46 46 81 83 83 F F F <---
51132024-02-03T17:24:43.024Z 6 120-139 45 45 45 81 81 81 F F F
51142024-02-03T17:24:43.024Z 7 140-159 45 46 46 81 84 84 F F F <---
51152024-02-03T17:24:43.024Z 8 160-179 44 44 44 80 80 80 F F F
51162024-02-03T17:24:43.024Z 9 180-199 43 46 46 78 83 83 F F F <---
51172024-02-03T17:24:43.024Z 10 200-219 42 46 46 77 83 83 F F F <---
51182024-02-03T17:24:43.024Z 11 220-239 45 45 45 81 81 81 F F F
51192024-02-03T17:24:43.024Z 12 240-259 45 46 46 81 84 84 T T T <---
51202024-02-03T17:24:43.025Z 13 260-279 44 44 44 79 79 79 F T T <---
51212024-02-03T17:24:43.025Z 14 280-299 41 41 41 74 74 74 F T T <---
51222024-02-03T17:24:43.025Z 15 300-319 38 38 38 67 67 67 F F F
51232024-02-03T17:24:43.025Z 16 320-339 45 46 46 81 84 84 F F F <---
51242024-02-03T17:24:43.025Z 17 340-359 40 40 40 71 71 71 F F F
51252024-02-03T17:24:43.025Z 18 360-379 39 39 39 70 70 70 F F F
51262024-02-03T17:24:43.025Z 19 380-399 34 34 34 62 62 62 F F F
51272024-02-03T17:24:43.025Z 20 400-419 42 42 42 76 76 76 F F F
51282024-02-03T17:24:43.025Z 21 420-439 45 45 45 81 81 81 F F F
51292024-02-03T17:24:43.025Z 22 440-459 45 45 45 81 81 81 F F F
51302024-02-03T17:24:43.025Z 23 460-479 39 39 39 70 70 70 F F F
51312024-02-03T17:24:43.026Z 24 480-499 45 46 46 81 84 84 F F F <---
51322024-02-03T17:24:43.026Z 25 500-519 42 42 42 75 75 75 F F F
51332024-02-03T17:24:43.026Z 26 520-539 42 42 42 77 77 77 F F F
51342024-02-03T17:24:43.026Z 27 540-559 45 45 45 81 81 81 F F F
51352024-02-03T17:24:43.026Z 28 560-579 45 46 46 81 84 84 F F F <---
51362024-02-03T17:24:43.026Z 29 580-599 40 46 46 71 84 84 F F F <---
51372024-02-03T17:24:43.026ZMax gen: 46, Max flush: 84
51382024-02-03T17:24:43.026ZError: Difference in extent metadata found!
51392024-02-03T17:24:43.026ZA Difference in extent metadata is expected here
51402024-02-03T17:24:43.026ZOn loop 23
51412024-02-03T17:24:43.026Z
51422024-02-03T17:24:43.030ZVerifying data now
51432024-02-03T17:24:48.162ZNo change after verify
51442024-02-03T17:24:48.162ZLoop: 23 Downstairs dump after verify (and repair):
51452024-02-03T17:24:48.190ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51462024-02-03T17:24:48.190Z 0 000-019 47 47 47 85 85 85 F F F
51472024-02-03T17:24:48.194Z 1 020-039 42 42 42 76 76 76 F F F
51482024-02-03T17:24:48.194Z 2 040-059 38 38 38 67 67 67 F F F
51492024-02-03T17:24:48.194Z 3 060-079 38 38 38 68 68 68 F F F
51502024-02-03T17:24:48.194Z 4 080-099 46 46 46 83 83 83 F F F
51512024-02-03T17:24:48.194Z 5 100-119 46 46 46 83 83 83 F F F
51522024-02-03T17:24:48.194Z 6 120-139 45 45 45 81 81 81 F F F
51532024-02-03T17:24:48.194Z 7 140-159 46 46 46 84 84 84 F F F
51542024-02-03T17:24:48.194Z 8 160-179 44 44 44 80 80 80 F F F
51552024-02-03T17:24:48.194Z 9 180-199 46 46 46 83 83 83 F F F
51562024-02-03T17:24:48.194Z 10 200-219 46 46 46 83 83 83 F F F
51572024-02-03T17:24:48.194Z 11 220-239 45 45 45 81 81 81 F F F
51582024-02-03T17:24:48.194Z 12 240-259 47 47 47 85 85 85 F F F
51592024-02-03T17:24:48.194Z 13 260-279 47 47 47 85 85 85 F F F
51602024-02-03T17:24:48.195Z 14 280-299 47 47 47 85 85 85 F F F
51612024-02-03T17:24:48.195Z 15 300-319 38 38 38 67 67 67 F F F
51622024-02-03T17:24:48.195Z 16 320-339 46 46 46 84 84 84 F F F
51632024-02-03T17:24:48.195Z 17 340-359 40 40 40 71 71 71 F F F
51642024-02-03T17:24:48.195Z 18 360-379 39 39 39 70 70 70 F F F
51652024-02-03T17:24:48.195Z 19 380-399 34 34 34 62 62 62 F F F
51662024-02-03T17:24:48.195Z 20 400-419 42 42 42 76 76 76 F F F
51672024-02-03T17:24:48.195Z 21 420-439 45 45 45 81 81 81 F F F
51682024-02-03T17:24:48.195Z 22 440-459 45 45 45 81 81 81 F F F
51692024-02-03T17:24:48.195Z 23 460-479 39 39 39 70 70 70 F F F
51702024-02-03T17:24:48.195Z 24 480-499 46 46 46 84 84 84 F F F
51712024-02-03T17:24:48.195Z 25 500-519 42 42 42 75 75 75 F F F
51722024-02-03T17:24:48.195Z 26 520-539 42 42 42 77 77 77 F F F
51732024-02-03T17:24:48.195Z 27 540-559 45 45 45 81 81 81 F F F
51742024-02-03T17:24:48.195Z 28 560-579 46 46 46 84 84 84 F F F
51752024-02-03T17:24:48.195Z 29 580-599 46 46 46 84 84 84 F F F
51762024-02-03T17:24:48.195ZMax gen: 47, Max flush: 85
51772024-02-03T17:24:48.195Z
51782024-02-03T17:24:48.195ZBegin loop 24
51792024-02-03T17:24:48.195ZDownstairs to restart: 0
51802024-02-03T17:24:48.224Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.384956049Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324}
51812024-02-03T17:24:48.225Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385439821Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324}
51822024-02-03T17:24:48.229Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385485395Z","hostname":"Crucible runtime is spawned
51832024-02-03T17:24:48.229Zip-10-150-1-93.us-west-2.compute.internal","pid":1324}
51842024-02-03T17:24:48.229Z{"msg":"Crucible cc0ccba0-f589-4b82-9716-7517075ba66f has session id: 77d1f95a-cf36-4135-8150-1bfc4b3eed8b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385621016Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51852024-02-03T17:24:48.229Z{"msg":"Upstairs opts: Upstairs UUID: cc0ccba0-f589-4b82-9716-7517075ba66f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385708699Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51862024-02-03T17:24:48.229Z{"msg":"Crucible stats registered with UUID: cc0ccba0-f589-4b82-9716-7517075ba66f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385744489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51872024-02-03T17:24:48.229Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385802164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324}
51882024-02-03T17:24:48.229Z{"msg":"cc0ccba0-f589-4b82-9716-7517075ba66f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385828994Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51892024-02-03T17:24:48.229Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385892652Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51902024-02-03T17:24:48.229Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.385939407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51912024-02-03T17:24:48.229Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386036906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51922024-02-03T17:24:48.230Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386130585Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51932024-02-03T17:24:48.230Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386158326Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51942024-02-03T17:24:48.230Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386192921Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51952024-02-03T17:24:48.230Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386285316Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"io task","client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51962024-02-03T17:24:48.230Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386357625Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"io task","client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51972024-02-03T17:24:48.230Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386402629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"io task","client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51982024-02-03T17:24:48.230Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386475915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"io task","client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
51992024-02-03T17:24:48.230Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386571844Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"io task","client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52002024-02-03T17:24:48.230Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.386617721Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"io task","client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52012024-02-03T17:24:48.230Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.387859775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52022024-02-03T17:24:48.230Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.387927141Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52032024-02-03T17:24:48.230Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.388000365Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52042024-02-03T17:24:48.230Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.388123098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52052024-02-03T17:24:48.231Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.388802981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52062024-02-03T17:24:48.231Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.388878665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52072024-02-03T17:24:48.235Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.389104949Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52082024-02-03T17:24:48.236Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.389190466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52092024-02-03T17:24:48.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.389939499Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52102024-02-03T17:24:48.236Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.390022768Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52112024-02-03T17:24:48.236Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.390167565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52122024-02-03T17:24:48.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.390191331Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52132024-02-03T17:24:48.236Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.390215181Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52142024-02-03T17:24:48.236Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.390258241Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52152024-02-03T17:24:48.236Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.390469047Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52162024-02-03T17:24:48.236Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391165876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52172024-02-03T17:24:48.236Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.39124679Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52182024-02-03T17:24:48.236Z{"msg":"[0]R flush_numbers[0..12]: [85, 76, 67, 68, 83, 83, 81, 84, 80, 83, 83, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391309059Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52192024-02-03T17:24:48.236Z{"msg":"[0]R generation[0..12]: [47, 42, 38, 38, 46, 46, 45, 46, 44, 46, 46, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391336117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52202024-02-03T17:24:48.236Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391379306Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52212024-02-03T17:24:48.237Z{"msg":"[1]R flush_numbers[0..12]: [85, 76, 67, 68, 83, 83, 81, 84, 80, 83, 83, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.39140466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52222024-02-03T17:24:48.237Z{"msg":"[1]R generation[0..12]: [47, 42, 38, 38, 46, 46, 45, 46, 44, 46, 46, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391447462Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52232024-02-03T17:24:48.237Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391472792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52242024-02-03T17:24:48.237Z{"Wait for a query_work_queue command to finish before sending IO
52252024-02-03T17:24:48.237Zmsg":"[2]R flush_numbers[0..12]: [85, 76, 67, 68, 83, 83, 81, 84, 80, 83, 83, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391499291Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52262024-02-03T17:24:48.237Z{"msg":"[2]R generation[0..12]: [47, 42, 38, 38, 46, 46, 45, 46, 44, 46, 46, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.39152189Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52272024-02-03T17:24:48.237Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391574114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52282024-02-03T17:24:48.237Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391599442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52292024-02-03T17:24:48.237Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391640872Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52302024-02-03T17:24:48.237Z{"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391663824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52312024-02-03T17:24:48.237Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391705877Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"":"downstairs","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52322024-02-03T17:24:48.237Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391729703Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52332024-02-03T17:24:48.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391749361Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"0","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52342024-02-03T17:24:48.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391789794Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"1","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52352024-02-03T17:24:48.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391813083Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"client":"2","":"downstairs","session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52362024-02-03T17:24:48.238Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391834543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52372024-02-03T17:24:48.238Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391876339Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52382024-02-03T17:24:48.238Z{"msg":"cc0ccba0-f589-4b82-9716-7517075ba66f is now active with session: 77d1f95a-cf36-4135-8150-1bfc4b3eed8b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391899777Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52392024-02-03T17:24:48.238Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391942255Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324,"session_id":"77d1f95a-cf36-4135-8150-1bfc4b3eed8b"}
52402024-02-03T17:24:48.238Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:47.391964353Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324}
52412024-02-03T17:24:48.238ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52422024-02-03T17:24:48.238ZLoading write count information from file "/tmp/test_repair_verify.data"
52432024-02-03T17:24:48.238ZRun Repair workload
52442024-02-03T17:24:48.238Z01/30 Write block 344 len 4096 data: 4
52452024-02-03T17:24:48.242Z02/30 Write block 504 len 32768 data: 6 7 9 7 7 7 5 3
52462024-02-03T17:24:48.242Z03/30 Write block 502 len 32768 data: 7 7 7 8 10 8 8 8
52472024-02-03T17:24:48.242Z04/30 Write block 420 len 32768 data: 5 3 3 3 3 3 2 2
52482024-02-03T17:24:48.243Z05/30 Read block 565 len 36864
52492024-02-03T17:24:48.243Z06/30 Read block 126 len 8192
52502024-02-03T17:24:48.243Z07/30 Read block 545 len 36864
52512024-02-03T17:24:48.246Z08/30 Read block 558 len 16384
52522024-02-03T17:24:48.247Z09/30 Write block 418 len 8192 data: 5 5
52532024-02-03T17:24:48.247Z10/30 Read block 382 len 12288
52542024-02-03T17:24:48.250Z11/30 Read block 243 len 12288
52552024-02-03T17:24:48.251Z12/30 Write block 207 len 16384 data: 5 3 3 5
52562024-02-03T17:24:48.251Z13/30 Read block 327 len 16384
52572024-02-03T17:24:48.254Z14/30 Read block 66 len 8192
52582024-02-03T17:24:48.254Z15/30 Write block 126 len 24576 data: 7 6 6 8 8 8
52592024-02-03T17:24:48.254Z16/30 Flush
52602024-02-03T17:24:48.258Z17/30 Write block 505 len 36864 data: 9 11 9 9 9 6 4 3 2
52612024-02-03T17:24:48.258Z18/30 Read block 155 len 40960
52622024-02-03T17:24:48.262Z19/30 Read block 561 len 24576
52632024-02-03T17:24:48.262Z20/30 Read block 62 len 40960
52642024-02-03T17:24:48.265Z21/30 Read block 524 len 40960
52652024-02-03T17:24:48.266Z22/30 Read block 527 len 24576
52662024-02-03T17:24:48.269Z23/30 Write block 219 len 28672 data: 6 5 8 8 5 5 6
52672024-02-03T17:24:48.269Z24/30 Write block 355 len 40960 data: 3 4 3 3 4 6 6 6 6 6
52682024-02-03T17:24:48.273Z25/30 Write block 548 len 28672 data: 5 5 5 7 6 6 6
52692024-02-03T17:24:48.273Z26/30 Flush
52702024-02-03T17:24:48.276Z27/30 Write block 195 len 36864 data: 3 4 3 3 5 6 4 4 4
52712024-02-03T17:24:48.280Z28/30 Read block 391 len 20480
52722024-02-03T17:24:48.280Z29/30 Read block 313 len 36864
52732024-02-03T17:24:48.284Z30/30 Write block 7 len 12288 data: 6 7 6
52742024-02-03T17:24:48.284Z----------------------------------------------------------------
52752024-02-03T17:24:48.284Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52762024-02-03T17:24:48.284ZStates: Active Active Active
52772024-02-03T17:24:48.284ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52782024-02-03T17:24:48.284Z 1 Acked 1000 Write 1 Sent Done Done false
52792024-02-03T17:24:48.284Z 2 Acked 1001 Write 8 Sent Done Done false
52802024-02-03T17:24:48.284Z 3 Acked 1002 Write 8 Sent Done Done false
52812024-02-03T17:24:48.284Z 4 Acked 1003 Write 8 Sent Done Done false
52822024-02-03T17:24:48.285Z 5 Acked 1004 Read 9 Sent Done Done false
52832024-02-03T17:24:48.285Z 6 Acked 1005 Read 2 Sent Done Done false
52842024-02-03T17:24:48.285Z 7 Acked 1006 Read 9 Sent Done Done false
52852024-02-03T17:24:48.285Z 8 Acked 1007 Read 4 Sent Done Done false
52862024-02-03T17:24:48.285Z 9 Acked 1008 Write 2 Sent Done Done false
52872024-02-03T17:24:48.285Z 10 Acked 1009 Read 3 Sent Done Done false
52882024-02-03T17:24:48.285Z 11 Acked 1010 Read 3 Sent Done Done false
52892024-02-03T17:24:48.285Z 12 Acked 1011 Write 4 Sent Done Done false
52902024-02-03T17:24:48.285Z 13 Acked 1012 Read 4 Sent Done Done false
52912024-02-03T17:24:48.285Z 14 Acked 1013 Read 2 Sent Done Done false
52922024-02-03T17:24:48.285Z 15 Acked 1014 Write 6 Sent Done Done false
52932024-02-03T17:24:48.285Z 16 Acked 1015 Flush 0 Sent Done Done false
52942024-02-03T17:24:48.285Z 17 Acked 1016 Write 9 Sent Done Done false
52952024-02-03T17:24:48.285Z 18 Acked 1017 Read 10 Sent Done Done false
52962024-02-03T17:24:48.285Z 19 Acked 1018 Read 6 Sent Done Done false
52972024-02-03T17:24:48.285Z 20 Acked 1019 Read 10 Sent Done Done false
52982024-02-03T17:24:48.286Z 21 Acked 1020 Read 10 Sent Done Done false
52992024-02-03T17:24:48.286Z 22 Acked 1021 Read 6 Sent Done Done false
53002024-02-03T17:24:48.286Z 23 Acked 1022 Write 7 Sent Done Done false
53012024-02-03T17:24:48.286Z 24 Acked 1023 Write 10 Sent Done Done false
53022024-02-03T17:24:48.286Z 25 Acked 1024 Write 7 Sent Done Done false
53032024-02-03T17:24:48.286Z 26 Acked 1025 Flush 0 Sent Done Done false
53042024-02-03T17:24:48.286Z 27 Acked 1026 Write 9 Sent Done Done false
53052024-02-03T17:24:48.286Z 28 Acked 1027 Read 5 Sent Done Done false
53062024-02-03T17:24:48.286Z 29 Acked 1028 Read 9 Sent Done Done false
53072024-02-03T17:24:48.286Z 30 Acked 1029 Write 3 Sent Sent Sent false
53082024-02-03T17:24:48.286Z STATES DS:0 DS:1 DS:2 TOTAL
53092024-02-03T17:24:48.286Z New 0 0 0 0
53102024-02-03T17:24:48.286Z Sent 30 1 1 32
53112024-02-03T17:24:48.286Z Done 0 29 29 58
53122024-02-03T17:24:48.287Z Skipped 0 0 0 0
53132024-02-03T17:24:48.287Z Error 0 0 0 0
53142024-02-03T17:24:48.287ZLast Flush: 0 1025 1025
53152024-02-03T17:24:48.287ZDownstairs last five completed:
53162024-02-03T17:24:48.287ZUpstairs last five completed: 30 29 28 27 26
53172024-02-03T17:24:48.287Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:47.443245674Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1324}
53182024-02-03T17:24:48.287ZWrote out file "/tmp/test_repair_verify.data"
53192024-02-03T17:24:48.290Z
53202024-02-03T17:24:48.298ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53212024-02-03T17:24:48.346ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53222024-02-03T17:24:48.346Z 0 000-019 47 47 47 85 85 85 F T T <---
53232024-02-03T17:24:48.346Z 1 020-039 42 42 42 76 76 76 F F F
53242024-02-03T17:24:48.346Z 2 040-059 38 38 38 67 67 67 F F F
53252024-02-03T17:24:48.346Z 3 060-079 38 38 38 68 68 68 F F F
53262024-02-03T17:24:48.347Z 4 080-099 46 46 46 83 83 83 F F F
53272024-02-03T17:24:48.352Z 5 100-119 46 46 46 83 83 83 F F F
53282024-02-03T17:24:48.352Z 6 120-139 45 48 48 81 86 86 F F F <---
53292024-02-03T17:24:48.353Z 7 140-159 46 46 46 84 84 84 F F F
53302024-02-03T17:24:48.353Z 8 160-179 44 44 44 80 80 80 F F F
53312024-02-03T17:24:48.353Z 9 180-199 46 46 46 83 83 83 F T T <---
53322024-02-03T17:24:48.353Z 10 200-219 46 48 48 83 87 87 F T T <---
53332024-02-03T17:24:48.353Z 11 220-239 45 48 48 81 87 87 F F F <---
53342024-02-03T17:24:48.353Z 12 240-259 47 47 47 85 85 85 F F F
53352024-02-03T17:24:48.353Z 13 260-279 47 47 47 85 85 85 F F F
53362024-02-03T17:24:48.353Z 14 280-299 47 47 47 85 85 85 F F F
53372024-02-03T17:24:48.354Z 15 300-319 38 38 38 67 67 67 F F F
53382024-02-03T17:24:48.354Z 16 320-339 46 46 46 84 84 84 F F F
53392024-02-03T17:24:48.354Z 17 340-359 40 48 48 71 87 87 F F F <---
53402024-02-03T17:24:48.354Z 18 360-379 39 48 48 70 87 87 F F F <---
53412024-02-03T17:24:48.354Z 19 380-399 34 34 34 62 62 62 F F F
53422024-02-03T17:24:48.354Z 20 400-419 42 48 48 76 86 86 F F F <---
53432024-02-03T17:24:48.354Z 21 420-439 45 48 48 81 86 86 F F F <---
53442024-02-03T17:24:48.355Z 22 440-459 45 45 45 81 81 81 F F F
53452024-02-03T17:24:48.355Z 23 460-479 39 39 39 70 70 70 F F F
53462024-02-03T17:24:48.355Z 24 480-499 46 46 46 84 84 84 F F F
53472024-02-03T17:24:48.355Z 25 500-519 42 48 48 75 87 87 F F F <---
53482024-02-03T17:24:48.355Z 26 520-539 42 42 42 77 77 77 F F F
53492024-02-03T17:24:48.355Z 27 540-559 45 48 48 81 87 87 F F F <---
53502024-02-03T17:24:48.355Z 28 560-579 46 46 46 84 84 84 F F F
53512024-02-03T17:24:48.355Z 29 580-599 46 46 46 84 84 84 F F F
53522024-02-03T17:24:48.355ZMax gen: 48, Max flush: 87
53532024-02-03T17:24:48.355ZError: Difference in extent metadata found!
53542024-02-03T17:24:48.355ZA Difference in extent metadata is expected here
53552024-02-03T17:24:48.355ZOn loop 24
53562024-02-03T17:24:48.355Z
53572024-02-03T17:24:48.356ZVerifying data now
53582024-02-03T17:24:53.316ZNo change after verify
53592024-02-03T17:24:53.316ZLoop: 24 Downstairs dump after verify (and repair):
53602024-02-03T17:24:53.340ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53612024-02-03T17:24:53.340Z 0 000-019 49 49 49 88 88 88 F F F
53622024-02-03T17:24:53.344Z 1 020-039 42 42 42 76 76 76 F F F
53632024-02-03T17:24:53.344Z 2 040-059 38 38 38 67 67 67 F F F
53642024-02-03T17:24:53.344Z 3 060-079 38 38 38 68 68 68 F F F
53652024-02-03T17:24:53.344Z 4 080-099 46 46 46 83 83 83 F F F
53662024-02-03T17:24:53.344Z 5 100-119 46 46 46 83 83 83 F F F
53672024-02-03T17:24:53.344Z 6 120-139 48 48 48 86 86 86 F F F
53682024-02-03T17:24:53.344Z 7 140-159 46 46 46 84 84 84 F F F
53692024-02-03T17:24:53.344Z 8 160-179 44 44 44 80 80 80 F F F
53702024-02-03T17:24:53.344Z 9 180-199 49 49 49 88 88 88 F F F
53712024-02-03T17:24:53.344Z 10 200-219 49 49 49 88 88 88 F F F
53722024-02-03T17:24:53.344Z 11 220-239 48 48 48 87 87 87 F F F
53732024-02-03T17:24:53.344Z 12 240-259 47 47 47 85 85 85 F F F
53742024-02-03T17:24:53.344Z 13 260-279 47 47 47 85 85 85 F F F
53752024-02-03T17:24:53.344Z 14 280-299 47 47 47 85 85 85 F F F
53762024-02-03T17:24:53.344Z 15 300-319 38 38 38 67 67 67 F F F
53772024-02-03T17:24:53.344Z 16 320-339 46 46 46 84 84 84 F F F
53782024-02-03T17:24:53.344Z 17 340-359 48 48 48 87 87 87 F F F
53792024-02-03T17:24:53.344Z 18 360-379 48 48 48 87 87 87 F F F
53802024-02-03T17:24:53.344Z 19 380-399 34 34 34 62 62 62 F F F
53812024-02-03T17:24:53.344Z 20 400-419 48 48 48 86 86 86 F F F
53822024-02-03T17:24:53.344Z 21 420-439 48 48 48 86 86 86 F F F
53832024-02-03T17:24:53.344Z 22 440-459 45 45 45 81 81 81 F F F
53842024-02-03T17:24:53.345Z 23 460-479 39 39 39 70 70 70 F F F
53852024-02-03T17:24:53.345Z 24 480-499 46 46 46 84 84 84 F F F
53862024-02-03T17:24:53.345Z 25 500-519 48 48 48 87 87 87 F F F
53872024-02-03T17:24:53.345Z 26 520-539 42 42 42 77 77 77 F F F
53882024-02-03T17:24:53.345Z 27 540-559 48 48 48 87 87 87 F F F
53892024-02-03T17:24:53.345Z 28 560-579 46 46 46 84 84 84 F F F
53902024-02-03T17:24:53.345Z 29 580-599 46 46 46 84 84 84 F F F
53912024-02-03T17:24:53.345ZMax gen: 49, Max flush: 88
53922024-02-03T17:24:53.348Z
53932024-02-03T17:24:53.348ZBegin loop 25
53942024-02-03T17:24:53.348ZDownstairs to restart: 1
53952024-02-03T17:24:53.380Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.540348394Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332}
53962024-02-03T17:24:53.380Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.540835604Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332}
53972024-02-03T17:24:53.380Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.540864495Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332}
53982024-02-03T17:24:53.384Z{"msg":"Crucible 63246743-4376-4176-b18c-46ade0a64647 has session id: 35f96c7c-438a-4e95-a311-26f43b3214a0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.540930281Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
53992024-02-03T17:24:53.384Z{Crucible runtime is spawned
54002024-02-03T17:24:53.384Z"msg":"Upstairs opts: Upstairs UUID: 63246743-4376-4176-b18c-46ade0a64647, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541034865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54012024-02-03T17:24:53.384Z{"msg":"Crucible stats registered with UUID: 63246743-4376-4176-b18c-46ade0a64647","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.54113551Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54022024-02-03T17:24:53.384Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541238818Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332}
54032024-02-03T17:24:53.384Z{"msg":"63246743-4376-4176-b18c-46ade0a64647 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541425596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54042024-02-03T17:24:53.384Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541481748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54052024-02-03T17:24:53.384Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541508328Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54062024-02-03T17:24:53.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541537338Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54072024-02-03T17:24:53.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541603971Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54082024-02-03T17:24:53.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541648649Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54092024-02-03T17:24:53.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541681053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54102024-02-03T17:24:53.385Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541727632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"io task","client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54112024-02-03T17:24:53.385Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541757879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"io task","client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54122024-02-03T17:24:53.385Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.541798817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"io task","client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54132024-02-03T17:24:53.385Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.542050279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"io task","client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54142024-02-03T17:24:53.385Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.542185785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"io task","client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54152024-02-03T17:24:53.385Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.542262929Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"io task","client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54162024-02-03T17:24:53.385Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.543664332Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54172024-02-03T17:24:53.385Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.543753986Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54182024-02-03T17:24:53.385Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.543869629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54192024-02-03T17:24:53.385Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.543893368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54202024-02-03T17:24:53.385Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.543937135Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54212024-02-03T17:24:53.385Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.543961613Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54222024-02-03T17:24:53.385Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.544650896Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54232024-02-03T17:24:53.385Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.544741234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54242024-02-03T17:24:53.389Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.54514596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54252024-02-03T17:24:53.389Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.545591145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54262024-02-03T17:24:53.389Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.545677044Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54272024-02-03T17:24:53.389Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.545744281Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54282024-02-03T17:24:53.389Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.545806807Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54292024-02-03T17:24:53.390Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.545902882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54302024-02-03T17:24:53.390Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.545949243Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54312024-02-03T17:24:53.390Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546042431Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54322024-02-03T17:24:53.390Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546138785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54332024-02-03T17:24:53.390Z{"msg":"[0]R flush_numbers[0..12]: [88, 76, 67, 68, 83, 83, 86, 84, 80, 88, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.5461748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54342024-02-03T17:24:53.390Z{"msg":"[0]R generation[0..12]: [49, 42, 38, 38, 46, 46, 48, 46, 44, 49, 49, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546201025Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54352024-02-03T17:24:53.390Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546225202Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54362024-02-03T17:24:53.390Z{"msg":"[1]R flush_numbers[0..12]: [88, 76, 67, 68, 83, 83, 86, 84, 80, 88, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546270923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54372024-02-03T17:24:53.390Z{"msg":"[1]R generation[0..12]: [49, 42, 38, 38, 46, 46, 48, 46, 44, 49, 49, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546296731Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54382024-02-03T17:24:53.390Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546320412Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54392024-02-03T17:24:53.390Z{"msg":"[2]R flush_numbers[0..12]: [88, 76, 67, 68, 83, 83, 86, 84, 80, 88, 88, 87]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546345138Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54402024-02-03T17:24:53.390Z{"msg":"[2]R generation[0..12]: [49, 42, 38, 38, 46, 46, 48, 46, 44, 49, 49, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546371368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54412024-02-03T17:24:53.390ZWait for a query_work_queue command to finish before sending IO
54422024-02-03T17:24:53.390Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546445719Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54432024-02-03T17:24:53.390Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546493528Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54442024-02-03T17:24:53.390Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546557137Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54452024-02-03T17:24:53.390Z{"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546581854Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54462024-02-03T17:24:53.390Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546603622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"":"downstairs","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54472024-02-03T17:24:53.390Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546625864Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54482024-02-03T17:24:53.390Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546646302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"0","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54492024-02-03T17:24:53.390Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546719151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"1","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54502024-02-03T17:24:53.390Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546751405Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"client":"2","":"downstairs","session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54512024-02-03T17:24:53.390Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546815044Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54522024-02-03T17:24:53.391Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546854427Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54532024-02-03T17:24:53.391Z{"msg":"63246743-4376-4176-b18c-46ade0a64647 is now active with session: 35f96c7c-438a-4e95-a311-26f43b3214a0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546901692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54542024-02-03T17:24:53.391Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546923768Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332,"session_id":"35f96c7c-438a-4e95-a311-26f43b3214a0"}
54552024-02-03T17:24:53.391Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:52.546988478Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332}
54562024-02-03T17:24:53.391ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54572024-02-03T17:24:53.391ZLoading write count information from file "/tmp/test_repair_verify.data"
54582024-02-03T17:24:53.391ZRun Repair workload
54592024-02-03T17:24:53.391Z01/30 Write block 201 len 32768 data: 5 5 5 3 4 4 6 4
54602024-02-03T17:24:53.394Z02/30 Write block 411 len 4096 data: 6
54612024-02-03T17:24:53.394Z03/30 Read block 34 len 4096
54622024-02-03T17:24:53.394Z04/30 Write block 109 len 28672 data: 6 6 7 7 7 8 6
54632024-02-03T17:24:53.394Z05/30 Write block 363 len 8192 data: 7 7
54642024-02-03T17:24:53.394Z06/30 Write block 227 len 8192 data: 6 5
54652024-02-03T17:24:53.394Z07/30 Read block 504 len 8192
54662024-02-03T17:24:53.397Z08/30 Write block 526 len 20480 data: 6 7 6 7 6
54672024-02-03T17:24:53.397Z09/30 Write block 356 len 16384 data: 5 4 4 5
54682024-02-03T17:24:53.397Z10/30 Read block 206 len 28672
54692024-02-03T17:24:53.400Z11/30 Write block 269 len 8192 data: 5 6
54702024-02-03T17:24:53.400Z12/30 Read block 287 len 8192
54712024-02-03T17:24:53.403Z13/30 Write block 270 len 40960 data: 7 8 8 8 8 10 11 8 6 4
54722024-02-03T17:24:53.403Z14/30 Write block 588 len 20480 data: 5 6 7 7 7
54732024-02-03T17:24:53.406Z15/30 Read block 299 len 20480
54742024-02-03T17:24:53.406Z16/30 Read block 41 len 40960
54752024-02-03T17:24:53.409Z17/30 Write block 196 len 4096 data: 5
54762024-02-03T17:24:53.409Z18/30 Read block 335 len 20480
54772024-02-03T17:24:53.409Z19/30 Read block 427 len 24576
54782024-02-03T17:24:53.412Z20/30 Read block 583 len 8192
54792024-02-03T17:24:53.412Z21/30 Write block 462 len 36864 data: 5 5 5 6 4 5 6 7 6
54802024-02-03T17:24:53.415Z22/30 Read block 492 len 20480
54812024-02-03T17:24:53.415Z23/30 Read block 418 len 28672
54822024-02-03T17:24:53.427Z24/30 Read block 91 len 28672
54832024-02-03T17:24:53.427Z25/30 Flush
54842024-02-03T17:24:53.431Z26/30 Flush
54852024-02-03T17:24:53.431Z27/30 Write block 278 len 24576 data: 7 5 3 5 4 4
54862024-02-03T17:24:53.431Z28/30 Write block 58 len 40960 data: 4 4 3 3 3 4 4 3 4 3
54872024-02-03T17:24:53.431Z29/30 Write block 370 len 12288 data: 4 4 4
54882024-02-03T17:24:53.431Z30/30 Write block 532 len 24576 data: 5 4 3 5 5 6
54892024-02-03T17:24:53.431Z----------------------------------------------------------------
54902024-02-03T17:24:53.432Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
54912024-02-03T17:24:53.432ZStates: Active Active Active
54922024-02-03T17:24:53.432ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54932024-02-03T17:24:53.432Z 1 Acked 1000 Write 8 Done Done Done false
54942024-02-03T17:24:53.432Z 2 Acked 1001 Write 1 Done Done Done false
54952024-02-03T17:24:53.432Z 3 Acked 1002 Read 1 Done Done Done false
54962024-02-03T17:24:53.432Z 4 Acked 1003 Write 7 Done Sent Done false
54972024-02-03T17:24:53.432Z 5 Acked 1004 Write 2 Done Sent Done false
54982024-02-03T17:24:53.432Z 6 Acked 1005 Write 2 Done Sent Done false
54992024-02-03T17:24:53.432Z 7 Acked 1006 Read 2 Done Sent Done false
55002024-02-03T17:24:53.432Z 8 Acked 1007 Write 5 Done Sent Done false
55012024-02-03T17:24:53.432Z 9 Acked 1008 Write 4 Done Sent Done false
55022024-02-03T17:24:53.432Z 10 Acked 1009 Read 7 Done Sent Done false
55032024-02-03T17:24:53.432Z 11 Acked 1010 Write 2 Done Sent Done false
55042024-02-03T17:24:53.432Z 12 Acked 1011 Read 2 Done Sent Done false
55052024-02-03T17:24:53.432Z 13 Acked 1012 Write 10 Done Sent Done false
55062024-02-03T17:24:53.432Z 14 Acked 1013 Write 5 Done Sent Done false
55072024-02-03T17:24:53.432Z 15 Acked 1014 Read 5 Done Sent Done false
55082024-02-03T17:24:53.432Z 16 Acked 1015 Read 10 Done Sent Done false
55092024-02-03T17:24:53.432Z 17 Acked 1016 Write 1 Done Sent Done false
55102024-02-03T17:24:53.432Z 18 Acked 1017 Read 5 Done Sent Done false
55112024-02-03T17:24:53.432Z 19 Acked 1018 Read 6 Done Sent Done false
55122024-02-03T17:24:53.432Z 20 Acked 1019 Read 2 Done Sent Done false
55132024-02-03T17:24:53.432Z 21 Acked 1020 Write 9 Done Sent Done false
55142024-02-03T17:24:53.433Z 22 Acked 1021 Read 5 Done Sent Done false
55152024-02-03T17:24:53.433Z 23 Acked 1022 Read 7 Done Sent Done false
55162024-02-03T17:24:53.433Z 24 Acked 1023 Read 7 Done Sent Done false
55172024-02-03T17:24:53.433Z 25 Acked 1024 Flush 0 Done Sent Done false
55182024-02-03T17:24:53.433Z 26 Acked 1025 Flush 0 Done Sent Done false
55192024-02-03T17:24:53.433Z 27 Acked 1026 Write 6 Done Sent Done false
55202024-02-03T17:24:53.433Z 28 Acked 1027 Write 10 Done Sent Done false
55212024-02-03T17:24:53.433Z 29 Acked 1028 Write 3 Sent Sent Done false
55222024-02-03T17:24:53.433Z 30 Acked 1029 Write 6 Sent Sent Sent false
55232024-02-03T17:24:53.433Z STATES DS:0 DS:1 DS:2 TOTAL
55242024-02-03T17:24:53.433Z New 0 0 0 0
55252024-02-03T17:24:53.433Z Sent 2 27 1 30
55262024-02-03T17:24:53.433Z Done 28 3 29 60
55272024-02-03T17:24:53.433Z Skipped 0 0 0 0
55282024-02-03T17:24:53.433Z Error 0 0 0 0
55292024-02-03T17:24:53.433ZLast Flush: 1025 0 1025
55302024-02-03T17:24:53.433ZDownstairs last five completed:
55312024-02-03T17:24:53.433ZUpstairs last five completed: 30 29 28 27 26
55322024-02-03T17:24:53.433Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:52.59060037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1332}
55332024-02-03T17:24:53.433ZWrote out file "/tmp/test_repair_verify.data"
55342024-02-03T17:24:53.437Z
55352024-02-03T17:24:53.445ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55362024-02-03T17:24:53.473ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55372024-02-03T17:24:53.473Z 0 000-019 49 49 49 88 88 88 F F F
55382024-02-03T17:24:53.473Z 1 020-039 42 42 42 76 76 76 F F F
55392024-02-03T17:24:53.473Z 2 040-059 38 38 38 67 67 67 T F T <---
55402024-02-03T17:24:53.477Z 3 060-079 38 38 38 68 68 68 T F T <---
55412024-02-03T17:24:53.477Z 4 080-099 46 46 46 83 83 83 F F F
55422024-02-03T17:24:53.477Z 5 100-119 50 46 50 89 83 89 F F F <---
55432024-02-03T17:24:53.477Z 6 120-139 48 48 48 86 86 86 F F F
55442024-02-03T17:24:53.477Z 7 140-159 46 46 46 84 84 84 F F F
55452024-02-03T17:24:53.477Z 8 160-179 44 44 44 80 80 80 F F F
55462024-02-03T17:24:53.477Z 9 180-199 50 49 50 89 88 89 F F F <---
55472024-02-03T17:24:53.477Z 10 200-219 50 49 50 89 88 89 F T F <---
55482024-02-03T17:24:53.477Z 11 220-239 50 48 50 89 87 89 F F F <---
55492024-02-03T17:24:53.477Z 12 240-259 47 47 47 85 85 85 F F F
55502024-02-03T17:24:53.477Z 13 260-279 50 47 50 89 85 89 T F T <---
55512024-02-03T17:24:53.477Z 14 280-299 47 47 47 85 85 85 T F T <---
55522024-02-03T17:24:53.477Z 15 300-319 38 38 38 67 67 67 F F F
55532024-02-03T17:24:53.477Z 16 320-339 46 46 46 84 84 84 F F F
55542024-02-03T17:24:53.477Z 17 340-359 50 48 50 89 87 89 F F F <---
55552024-02-03T17:24:53.477Z 18 360-379 50 48 50 89 87 89 T F T <---
55562024-02-03T17:24:53.477Z 19 380-399 34 34 34 62 62 62 F F F
55572024-02-03T17:24:53.477Z 20 400-419 50 48 50 89 86 89 F T F <---
55582024-02-03T17:24:53.477Z 21 420-439 48 48 48 86 86 86 F F F
55592024-02-03T17:24:53.477Z 22 440-459 45 45 45 81 81 81 F F F
55602024-02-03T17:24:53.477Z 23 460-479 50 39 50 89 70 89 F F F <---
55612024-02-03T17:24:53.477Z 24 480-499 46 46 46 84 84 84 F F F
55622024-02-03T17:24:53.478Z 25 500-519 48 48 48 87 87 87 F F F
55632024-02-03T17:24:53.478Z 26 520-539 50 42 50 89 77 89 T F T <---
55642024-02-03T17:24:53.478Z 27 540-559 48 48 48 87 87 87 F F F
55652024-02-03T17:24:53.478Z 28 560-579 46 46 46 84 84 84 F F F
55662024-02-03T17:24:53.478Z 29 580-599 50 46 50 89 84 89 F F F <---
55672024-02-03T17:24:53.478ZMax gen: 50, Max flush: 89
55682024-02-03T17:24:53.478ZError: Difference in extent metadata found!
55692024-02-03T17:24:53.481ZA Difference in extent metadata is expected here
55702024-02-03T17:24:53.481ZOn loop 25
55712024-02-03T17:24:53.481Z
55722024-02-03T17:24:53.487ZVerifying data now
55732024-02-03T17:24:58.714ZNo change after verify
55742024-02-03T17:24:58.714ZLoop: 25 Downstairs dump after verify (and repair):
55752024-02-03T17:24:58.738ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55762024-02-03T17:24:58.739Z 0 000-019 49 49 49 88 88 88 F F F
55772024-02-03T17:24:58.742Z 1 020-039 42 42 42 76 76 76 F F F
55782024-02-03T17:24:58.742Z 2 040-059 51 51 51 90 90 90 F F F
55792024-02-03T17:24:58.742Z 3 060-079 51 51 51 90 90 90 F F F
55802024-02-03T17:24:58.742Z 4 080-099 46 46 46 83 83 83 F F F
55812024-02-03T17:24:58.742Z 5 100-119 50 50 50 89 89 89 F F F
55822024-02-03T17:24:58.742Z 6 120-139 48 48 48 86 86 86 F F F
55832024-02-03T17:24:58.742Z 7 140-159 46 46 46 84 84 84 F F F
55842024-02-03T17:24:58.742Z 8 160-179 44 44 44 80 80 80 F F F
55852024-02-03T17:24:58.742Z 9 180-199 50 50 50 89 89 89 F F F
55862024-02-03T17:24:58.742Z 10 200-219 50 50 50 89 89 89 F F F
55872024-02-03T17:24:58.742Z 11 220-239 50 50 50 89 89 89 F F F
55882024-02-03T17:24:58.742Z 12 240-259 47 47 47 85 85 85 F F F
55892024-02-03T17:24:58.742Z 13 260-279 51 51 51 90 90 90 F F F
55902024-02-03T17:24:58.742Z 14 280-299 51 51 51 90 90 90 F F F
55912024-02-03T17:24:58.742Z 15 300-319 38 38 38 67 67 67 F F F
55922024-02-03T17:24:58.742Z 16 320-339 46 46 46 84 84 84 F F F
55932024-02-03T17:24:58.742Z 17 340-359 50 50 50 89 89 89 F F F
55942024-02-03T17:24:58.742Z 18 360-379 51 51 51 90 90 90 F F F
55952024-02-03T17:24:58.742Z 19 380-399 34 34 34 62 62 62 F F F
55962024-02-03T17:24:58.743Z 20 400-419 50 50 50 89 89 89 F F F
55972024-02-03T17:24:58.743Z 21 420-439 48 48 48 86 86 86 F F F
55982024-02-03T17:24:58.743Z 22 440-459 45 45 45 81 81 81 F F F
55992024-02-03T17:24:58.743Z 23 460-479 50 50 50 89 89 89 F F F
56002024-02-03T17:24:58.743Z 24 480-499 46 46 46 84 84 84 F F F
56012024-02-03T17:24:58.743Z 25 500-519 48 48 48 87 87 87 F F F
56022024-02-03T17:24:58.743Z 26 520-539 51 51 51 90 90 90 F F F
56032024-02-03T17:24:58.743Z 27 540-559 48 48 48 87 87 87 F F F
56042024-02-03T17:24:58.743Z 28 560-579 46 46 46 84 84 84 F F F
56052024-02-03T17:24:58.743Z 29 580-599 50 50 50 89 89 89 F F F
56062024-02-03T17:24:58.743ZMax gen: 51, Max flush: 90
56072024-02-03T17:24:58.749Z
56082024-02-03T17:24:58.749ZBegin loop 26
56092024-02-03T17:24:58.749ZDownstairs to restart: 2
56102024-02-03T17:24:58.778Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.938789092Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340}
56112024-02-03T17:24:58.778Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.939465472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340}
56122024-02-03T17:24:58.782Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.939505115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340}
56132024-02-03T17:24:58.782Z{"msg":"Crucible 5581e9dd-bb6a-4d47-844c-c018dd7add2f has session id: b109689e-cd96-48b9-9592-acfb6d99655f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.93956652Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"Crucible runtime is spawned
56142024-02-03T17:24:58.782Zb109689e-cd96-48b9-9592-acfb6d99655f"}
56152024-02-03T17:24:58.782Z{"msg":"Upstairs opts: Upstairs UUID: 5581e9dd-bb6a-4d47-844c-c018dd7add2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.93970557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56162024-02-03T17:24:58.782Z{"msg":"Crucible stats registered with UUID: 5581e9dd-bb6a-4d47-844c-c018dd7add2f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.939776488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56172024-02-03T17:24:58.782Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.939802829Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340}
56182024-02-03T17:24:58.782Z{"msg":"5581e9dd-bb6a-4d47-844c-c018dd7add2f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.939858114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56192024-02-03T17:24:58.782Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.939884413Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56202024-02-03T17:24:58.782Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.939943344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56212024-02-03T17:24:58.782Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.939984841Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56222024-02-03T17:24:58.782Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.940028733Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56232024-02-03T17:24:58.782Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.940163815Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56242024-02-03T17:24:58.782Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.940240406Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56252024-02-03T17:24:58.782Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.940370609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"io task","client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56262024-02-03T17:24:58.783Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.940449219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"io task","client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56272024-02-03T17:24:58.783Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.940520511Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"io task","client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56282024-02-03T17:24:58.783Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.940563633Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"io task","client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56292024-02-03T17:24:58.783Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.940632817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"io task","client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56302024-02-03T17:24:58.783Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.94070876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"io task","client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56312024-02-03T17:24:58.783Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.941827397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56322024-02-03T17:24:58.783Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.941893246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56332024-02-03T17:24:58.783Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.941919696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56342024-02-03T17:24:58.783Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.942024869Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56352024-02-03T17:24:58.783Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.942567173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56362024-02-03T17:24:58.783Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.942643879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56372024-02-03T17:24:58.783Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.942775688Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56382024-02-03T17:24:58.787Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.943173632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56392024-02-03T17:24:58.787Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.943756991Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56402024-02-03T17:24:58.787Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.943816728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56412024-02-03T17:24:58.787Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.943842389Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56422024-02-03T17:24:58.787Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.943899715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56432024-02-03T17:24:58.788Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944116328Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56442024-02-03T17:24:58.788Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944177314Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56452024-02-03T17:24:58.788Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944272187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56462024-02-03T17:24:58.788Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944642702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56472024-02-03T17:24:58.788Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944744041Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56482024-02-03T17:24:58.788Z{"msg":"[0]R flush_numbers[0..12]: [88, 76, 90, 90, 83, 89, 86, 84, 80, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944779402Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56492024-02-03T17:24:58.788Z{"msg":"[0]R generation[0..12]: [49, 42, 51, 51, 46, 50, 48, 46, 44, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944840787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56502024-02-03T17:24:58.788Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944869942Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56512024-02-03T17:24:58.788Z{"msg":"[1]R flush_numbers[0..12]: [88, 76, 90, 90, 83, 89, 86, 84, 80, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944893028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56522024-02-03T17:24:58.788Z{"msg":"[1]R generation[0..12]: [49, 42, 51, 51, 46, 50, 48, 46, 44, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944921791Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56532024-02-03T17:24:58.788ZWait for a query_work_queue command to finish before sending IO
56542024-02-03T17:24:58.788Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.944984124Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56552024-02-03T17:24:58.788Z{"msg":"[2]R flush_numbers[0..12]: [88, 76, 90, 90, 83, 89, 86, 84, 80, 89, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945016353Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56562024-02-03T17:24:58.788Z{"msg":"[2]R generation[0..12]: [49, 42, 51, 51, 46, 50, 48, 46, 44, 50, 50, 50]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945082397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56572024-02-03T17:24:58.788Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945116611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56582024-02-03T17:24:58.788Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945175839Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56592024-02-03T17:24:58.788Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945202457Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56602024-02-03T17:24:58.788Z{"msg":"Next flush: 91","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945267941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56612024-02-03T17:24:58.788Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945294801Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"":"downstairs","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56622024-02-03T17:24:58.788Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945360577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56632024-02-03T17:24:58.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945391852Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"0","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56642024-02-03T17:24:58.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945450939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"1","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56652024-02-03T17:24:58.788Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945481231Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"client":"2","":"downstairs","session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56662024-02-03T17:24:58.788Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56672024-02-03T17:24:58.789Z,"time":"2024-02-03T17:24:57.94554234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56682024-02-03T17:24:58.789Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945571076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56692024-02-03T17:24:58.789Z{"msg":"5581e9dd-bb6a-4d47-844c-c018dd7add2f is now active with session: b109689e-cd96-48b9-9592-acfb6d99655f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945626469Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56702024-02-03T17:24:58.789Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945655059Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340,"session_id":"b109689e-cd96-48b9-9592-acfb6d99655f"}
56712024-02-03T17:24:58.789Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:24:57.945709439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340}
56722024-02-03T17:24:58.789ZLoading write count information from file "/tmp/test_repair_verify.data"
56732024-02-03T17:24:58.789ZRun Repair workload
56742024-02-03T17:24:58.789Z01/30 Write block 261 len 24576 data: 6 5 4 4 3 4
56752024-02-03T17:24:58.792Z02/30 Read block 313 len 16384
56762024-02-03T17:24:58.792Z03/30 Read block 381 len 36864
56772024-02-03T17:24:58.792Z04/30 Read block 341 len 12288
56782024-02-03T17:24:58.795Z05/30 Read block 530 len 28672
56792024-02-03T17:24:58.795Z06/30 Read block 83 len 24576
56802024-02-03T17:24:58.798Z07/30 Write block 268 len 28672 data: 5 6 8 9 9 9 9
56812024-02-03T17:24:58.798Z08/30 Write block 58 len 20480 data: 5 5 4 4 4
56822024-02-03T17:24:58.798Z09/30 Write block 375 len 8192 data: 3 4
56832024-02-03T17:24:58.801Z10/30 Write block 361 len 24576 data: 7 7 8 8 7 5
56842024-02-03T17:24:58.801Z11/30 Read block 187 len 12288
56852024-02-03T17:24:58.801Z12/30 Read block 494 len 24576
56862024-02-03T17:24:58.804Z13/30 Write block 301 len 12288 data: 3 3 3
56872024-02-03T17:24:58.805Z14/30 Read block 191 len 40960
56882024-02-03T17:24:58.807Z15/30 Write block 221 len 40960 data: 9 9 6 6 7 5 7 6 3 3
56892024-02-03T17:24:58.807Z16/30 Read block 153 len 32768
56902024-02-03T17:24:58.810Z17/30 Flush
56912024-02-03T17:24:58.814Z18/30 Write block 59 len 28672 data: 6 5 5 5 5 5 4
56922024-02-03T17:24:58.817Z19/30 Read block 355 len 20480
56932024-02-03T17:24:58.817Z20/30 Flush
56942024-02-03T17:24:58.820Z21/30 Write block 388 len 20480 data: 4 4 2 3 3
56952024-02-03T17:24:58.820Z22/30 Read block 575 len 16384
56962024-02-03T17:24:58.823Z23/30 Flush
56972024-02-03T17:24:58.826Z24/30 Write block 517 len 8192 data: 2 2
56982024-02-03T17:24:58.826Z25/30 Read block 347 len 20480
56992024-02-03T17:24:58.826Z26/30 Read block 480 len 20480
57002024-02-03T17:24:58.832Z27/30 Write block 180 len 16384 data: 5 6 5 4
57012024-02-03T17:24:58.832Z28/30 Read block 116 len 36864
57022024-02-03T17:24:58.835Z29/30 Write block 514 len 8192 data: 2 2
57032024-02-03T17:24:58.835Z30/30 Write block 10 len 8192 data: 6 6
57042024-02-03T17:24:58.835Z----------------------------------------------------------------
57052024-02-03T17:24:58.835Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57062024-02-03T17:24:58.835ZStates: Active Active Active
57072024-02-03T17:24:58.835ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57082024-02-03T17:24:58.836Z 1 Acked 1000 Write 6 Done Done Sent false
57092024-02-03T17:24:58.836Z 2 Acked 1001 Read 4 Done Done Sent false
57102024-02-03T17:24:58.836Z 3 Acked 1002 Read 9 Done Done Sent false
57112024-02-03T17:24:58.836Z 4 Acked 1003 Read 3 Done Done Sent false
57122024-02-03T17:24:58.836Z 5 Acked 1004 Read 7 Done Done Sent false
57132024-02-03T17:24:58.836Z 6 Acked 1005 Read 6 Done Done Sent false
57142024-02-03T17:24:58.836Z 7 Acked 1006 Write 7 Done Done Sent false
57152024-02-03T17:24:58.836Z 8 Acked 1007 Write 5 Done Done Sent false
57162024-02-03T17:24:58.836Z 9 Acked 1008 Write 2 Done Done Sent false
57172024-02-03T17:24:58.836Z 10 Acked 1009 Write 6 Done Done Sent false
57182024-02-03T17:24:58.836Z 11 Acked 1010 Read 3 Done Done Sent false
57192024-02-03T17:24:58.836Z 12 Acked 1011 Read 6 Done Done Sent false
57202024-02-03T17:24:58.836Z 13 Acked 1012 Write 3 Done Done Sent false
57212024-02-03T17:24:58.836Z 14 Acked 1013 Read 10 Done Done Sent false
57222024-02-03T17:24:58.836Z 15 Acked 1014 Write 10 Done Done Sent false
57232024-02-03T17:24:58.836Z 16 Acked 1015 Read 8 Done Done Sent false
57242024-02-03T17:24:58.836Z 17 Acked 1016 Flush 0 Done Done Sent false
57252024-02-03T17:24:58.836Z 18 Acked 1017 Write 7 Done Done Sent false
57262024-02-03T17:24:58.836Z 19 Acked 1018 Read 5 Done Done Sent false
57272024-02-03T17:24:58.836Z 20 Acked 1019 Flush 0 Done Done Sent false
57282024-02-03T17:24:58.836Z 21 Acked 1020 Write 5 Done Done Sent false
57292024-02-03T17:24:58.836Z 22 Acked 1021 Read 4 Done Done Sent false
57302024-02-03T17:24:58.836Z 23 Acked 1022 Flush 0 Done Done Sent false
57312024-02-03T17:24:58.837Z 24 Acked 1023 Write 2 Done Done Sent false
57322024-02-03T17:24:58.837Z 25 Acked 1024 Read 5 Done Done Sent false
57332024-02-03T17:24:58.837Z 26 Acked 1025 Read 5 Done Done Sent false
57342024-02-03T17:24:58.837Z 27 Acked 1026 Write 4 Done Done Sent false
57352024-02-03T17:24:58.837Z 28 Acked 1027 Read 9 Done Done Sent false
57362024-02-03T17:24:58.837Z 29 Acked 1028 Write 2 Done Done Sent false
57372024-02-03T17:24:58.837Z 30 Acked 1029 Write 2 Sent Sent Sent false
57382024-02-03T17:24:58.837Z STATES DS:0 DS:1 DS:2 TOTAL
57392024-02-03T17:24:58.837Z New 0 0 0 0
57402024-02-03T17:24:58.837Z Sent 1 1 30 32
57412024-02-03T17:24:58.837Z Done 29 29 0 58
57422024-02-03T17:24:58.837Z Skipped 0 0 0 0
57432024-02-03T17:24:58.837Z Error 0 0 0 0
57442024-02-03T17:24:58.837ZLast Flush: 1022 1022 0
57452024-02-03T17:24:58.837ZDownstairs last five completed:
57462024-02-03T17:24:58.837ZUpstairs last five completed: 30 29 28 27 26
57472024-02-03T17:24:58.837Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:24:57.993174103Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1340}
57482024-02-03T17:24:58.837ZWrote out file "/tmp/test_repair_verify.data"
57492024-02-03T17:24:58.841Z
57502024-02-03T17:24:58.847ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57512024-02-03T17:24:58.873ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57522024-02-03T17:24:58.873Z 0 000-019 49 49 49 88 88 88 T T F <---
57532024-02-03T17:24:58.876Z 1 020-039 42 42 42 76 76 76 F F F
57542024-02-03T17:24:58.876Z 2 040-059 52 52 51 92 92 90 F F F <---
57552024-02-03T17:24:58.876Z 3 060-079 52 52 51 92 92 90 F F F <---
57562024-02-03T17:24:58.876Z 4 080-099 46 46 46 83 83 83 F F F
57572024-02-03T17:24:58.876Z 5 100-119 50 50 50 89 89 89 F F F
57582024-02-03T17:24:58.877Z 6 120-139 48 48 48 86 86 86 F F F
57592024-02-03T17:24:58.877Z 7 140-159 46 46 46 84 84 84 F F F
57602024-02-03T17:24:58.877Z 8 160-179 44 44 44 80 80 80 F F F
57612024-02-03T17:24:58.877Z 9 180-199 50 50 50 89 89 89 T T F <---
57622024-02-03T17:24:58.877Z 10 200-219 50 50 50 89 89 89 F F F
57632024-02-03T17:24:58.877Z 11 220-239 52 52 50 91 91 89 F F F <---
57642024-02-03T17:24:58.877Z 12 240-259 47 47 47 85 85 85 F F F
57652024-02-03T17:24:58.877Z 13 260-279 52 52 51 91 91 90 F F F <---
57662024-02-03T17:24:58.877Z 14 280-299 51 51 51 90 90 90 F F F
57672024-02-03T17:24:58.877Z 15 300-319 52 52 38 91 91 67 F F F <---
57682024-02-03T17:24:58.877Z 16 320-339 46 46 46 84 84 84 F F F
57692024-02-03T17:24:58.877Z 17 340-359 50 50 50 89 89 89 F F F
57702024-02-03T17:24:58.877Z 18 360-379 52 52 51 91 91 90 F F F <---
57712024-02-03T17:24:58.877Z 19 380-399 52 52 34 93 93 62 F F F <---
57722024-02-03T17:24:58.877Z 20 400-419 50 50 50 89 89 89 F F F
57732024-02-03T17:24:58.877Z 21 420-439 48 48 48 86 86 86 F F F
57742024-02-03T17:24:58.877Z 22 440-459 45 45 45 81 81 81 F F F
57752024-02-03T17:24:58.877Z 23 460-479 50 50 50 89 89 89 F F F
57762024-02-03T17:24:58.877Z 24 480-499 46 46 46 84 84 84 F F F
57772024-02-03T17:24:58.877Z 25 500-519 48 48 48 87 87 87 T T F <---
57782024-02-03T17:24:58.877Z 26 520-539 51 51 51 90 90 90 F F F
57792024-02-03T17:24:58.877Z 27 540-559 48 48 48 87 87 87 F F F
57802024-02-03T17:24:58.877Z 28 560-579 46 46 46 84 84 84 F F F
57812024-02-03T17:24:58.878Z 29 580-599 50 50 50 89 89 89 F F F
57822024-02-03T17:24:58.878ZMax gen: 52, Max flush: 93
57832024-02-03T17:24:58.878ZError: Difference in extent metadata found!
57842024-02-03T17:24:58.881ZA Difference in extent metadata is expected here
57852024-02-03T17:24:58.881ZOn loop 26
57862024-02-03T17:24:58.881Z
57872024-02-03T17:24:58.881ZVerifying data now
57882024-02-03T17:25:03.798ZNo change after verify
57892024-02-03T17:25:03.798ZLoop: 26 Downstairs dump after verify (and repair):
57902024-02-03T17:25:03.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57912024-02-03T17:25:03.824Z 0 000-019 53 53 53 94 94 94 F F F
57922024-02-03T17:25:03.829Z 1 020-039 42 42 42 76 76 76 F F F
57932024-02-03T17:25:03.829Z 2 040-059 52 52 52 92 92 92 F F F
57942024-02-03T17:25:03.829Z 3 060-079 52 52 52 92 92 92 F F F
57952024-02-03T17:25:03.829Z 4 080-099 46 46 46 83 83 83 F F F
57962024-02-03T17:25:03.829Z 5 100-119 50 50 50 89 89 89 F F F
57972024-02-03T17:25:03.829Z 6 120-139 48 48 48 86 86 86 F F F
57982024-02-03T17:25:03.829Z 7 140-159 46 46 46 84 84 84 F F F
57992024-02-03T17:25:03.830Z 8 160-179 44 44 44 80 80 80 F F F
58002024-02-03T17:25:03.830Z 9 180-199 53 53 53 94 94 94 F F F
58012024-02-03T17:25:03.830Z 10 200-219 50 50 50 89 89 89 F F F
58022024-02-03T17:25:03.830Z 11 220-239 52 52 52 91 91 91 F F F
58032024-02-03T17:25:03.830Z 12 240-259 47 47 47 85 85 85 F F F
58042024-02-03T17:25:03.830Z 13 260-279 52 52 52 91 91 91 F F F
58052024-02-03T17:25:03.830Z 14 280-299 51 51 51 90 90 90 F F F
58062024-02-03T17:25:03.830Z 15 300-319 52 52 52 91 91 91 F F F
58072024-02-03T17:25:03.830Z 16 320-339 46 46 46 84 84 84 F F F
58082024-02-03T17:25:03.830Z 17 340-359 50 50 50 89 89 89 F F F
58092024-02-03T17:25:03.830Z 18 360-379 52 52 52 91 91 91 F F F
58102024-02-03T17:25:03.830Z 19 380-399 52 52 52 93 93 93 F F F
58112024-02-03T17:25:03.831Z 20 400-419 50 50 50 89 89 89 F F F
58122024-02-03T17:25:03.831Z 21 420-439 48 48 48 86 86 86 F F F
58132024-02-03T17:25:03.831Z 22 440-459 45 45 45 81 81 81 F F F
58142024-02-03T17:25:03.831Z 23 460-479 50 50 50 89 89 89 F F F
58152024-02-03T17:25:03.831Z 24 480-499 46 46 46 84 84 84 F F F
58162024-02-03T17:25:03.831Z 25 500-519 53 53 53 94 94 94 F F F
58172024-02-03T17:25:03.831Z 26 520-539 51 51 51 90 90 90 F F F
58182024-02-03T17:25:03.831Z 27 540-559 48 48 48 87 87 87 F F F
58192024-02-03T17:25:03.831Z 28 560-579 46 46 46 84 84 84 F F F
58202024-02-03T17:25:03.831Z 29 580-599 50 50 50 89 89 89 F F F
58212024-02-03T17:25:03.831ZMax gen: 53, Max flush: 94
58222024-02-03T17:25:03.831Z
58232024-02-03T17:25:03.831ZBegin loop 27
58242024-02-03T17:25:03.831ZDownstairs to restart: 1
58252024-02-03T17:25:03.861Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.022047895Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348}
58262024-02-03T17:25:03.862Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.022577944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348}
58272024-02-03T17:25:03.867Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.022656083Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348}
58282024-02-03T17:25:03.867Z{Crucible runtime is spawned
58292024-02-03T17:25:03.867Z"msg":"Crucible 5c0102a3-22f2-45d0-a5d3-e182419dd95c has session id: b889a03f-236d-4502-afca-c32d5fd4851f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.022750317Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58302024-02-03T17:25:03.867Z{"msg":"Upstairs opts: Upstairs UUID: 5c0102a3-22f2-45d0-a5d3-e182419dd95c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.022853267Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58312024-02-03T17:25:03.867Z{"msg":"Crucible stats registered with UUID: 5c0102a3-22f2-45d0-a5d3-e182419dd95c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.022924085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58322024-02-03T17:25:03.867Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.022966567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348}
58332024-02-03T17:25:03.867Z{"msg":"5c0102a3-22f2-45d0-a5d3-e182419dd95c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023002079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58342024-02-03T17:25:03.867Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023071681Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58352024-02-03T17:25:03.867Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.02311629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58362024-02-03T17:25:03.867Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023190625Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58372024-02-03T17:25:03.867Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.02325321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58382024-02-03T17:25:03.868Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023297588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58392024-02-03T17:25:03.868Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023385147Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58402024-02-03T17:25:03.868Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023430036Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58412024-02-03T17:25:03.868Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.02352022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58422024-02-03T17:25:03.868Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023609611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58432024-02-03T17:25:03.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023795492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"io task","client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58442024-02-03T17:25:03.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023905943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"io task","client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58452024-02-03T17:25:03.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.023951114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"io task","client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58462024-02-03T17:25:03.868Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.025324896Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58472024-02-03T17:25:03.868Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.025440757Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58482024-02-03T17:25:03.868Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.02554112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58492024-02-03T17:25:03.868Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.025581672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58502024-02-03T17:25:03.869Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.026402294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58512024-02-03T17:25:03.869Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.02648346Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58522024-02-03T17:25:03.869Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.026590214Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58532024-02-03T17:25:03.869Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.026689565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58542024-02-03T17:25:03.869Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.02737059Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58552024-02-03T17:25:03.869Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.027450312Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58562024-02-03T17:25:03.878Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.02749067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58572024-02-03T17:25:03.878Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.027555855Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58582024-02-03T17:25:03.878Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.0276262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58592024-02-03T17:25:03.878Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.027659841Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58602024-02-03T17:25:03.878Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.027962716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58612024-02-03T17:25:03.878Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.028728675Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58622024-02-03T17:25:03.878Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.028798424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58632024-02-03T17:25:03.878Z{"msg":"[0]R flush_numbers[0..12]: [94, 76, 92, 92, 83, 89, 86, 84, 80, 94, 89, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.028826572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58642024-02-03T17:25:03.878Z{"msg":"[0]R generation[0..12]: [53, 42, 52, 52, 46, 50, 48, 46, 44, 53, 50, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.028850444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58652024-02-03T17:25:03.878Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.028874824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58662024-02-03T17:25:03.879Z{"msg":"[1]R flush_numbers[0..12]: [94, 76, 92, 92, 83, 89, 86, 84, 80, 94, 89, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.028917651Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58672024-02-03T17:25:03.879Z{"msg":"[1]R generation[0..12]: [53, 42, 52, 52, 46, 50, 48, 46, 44, 53, 50, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.028943294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58682024-02-03T17:25:03.879Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.028985284Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58692024-02-03T17:25:03.879Z{"msg":"[2]R flush_numbers[0..12]: [94, 76, 92, 92, 83, 89, 86, 84, 80, 94, 89, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029010766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58702024-02-03T17:25:03.879Z{"msg":"[2]R generation[0..12]: [53, 42, 52, 52, 46, 50, 48, 46, 44, 53, 50, 52]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
58712024-02-03T17:25:03.879Z,"time":"2024-02-03T17:25:03.029099438Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58722024-02-03T17:25:03.879Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029212722Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58732024-02-03T17:25:03.879Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029243439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58742024-02-03T17:25:03.879Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029299484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58752024-02-03T17:25:03.879Z{"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029322686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58762024-02-03T17:25:03.879Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029345685Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"":"downstairs","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58772024-02-03T17:25:03.879Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029367891Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58782024-02-03T17:25:03.879Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"0","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58792024-02-03T17:25:03.879Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029432407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"1","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58802024-02-03T17:25:03.879Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029456226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"client":"2","":"downstairs","session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58812024-02-03T17:25:03.880Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029478337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58822024-02-03T17:25:03.880Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029536947Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58832024-02-03T17:25:03.880Z{"msg":"5c0102a3-22f2-45d0-a5d3-e182419dd95c is now active with session: b889a03f-236d-4502-afca-c32d5fd4851f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029564094Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58842024-02-03T17:25:03.880Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029583973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348,"session_id":"b889a03f-236d-4502-afca-c32d5fd4851f"}
58852024-02-03T17:25:03.880Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:03.029604828Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348}
58862024-02-03T17:25:03.880ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58872024-02-03T17:25:03.880ZLoading write count information from file "/tmp/test_repair_verify.data"
58882024-02-03T17:25:03.880ZRun Repair workload
58892024-02-03T17:25:03.880Z01/30 Write block 284 len 8192 data: 5 4
58902024-02-03T17:25:03.880Z02/30 Read block 493 len 40960
58912024-02-03T17:25:03.880Z03/30 Write block 40 len 28672 data: 5 5 4 4 5 4 4
58922024-02-03T17:25:03.885Z04/30 Read block 332 len 32768
58932024-02-03T17:25:03.885Z05/30 Write block 227 len 40960 data: 8 7 4 4 4 4 4 5 4 5
58942024-02-03T17:25:03.885Z06/30 Write block 301 len 20480 data: 4 4 4 3 3
58952024-02-03T17:25:03.885Z07/30 Write block 121 len 12288 data: 5 4 5
58962024-02-03T17:25:03.886Z08/30 Read block 201 len 20480
58972024-02-03T17:25:03.886Z09/30 Write block 276 len 12288 data: 12 9 8
58982024-02-03T17:25:03.886Z10/30 Read block 581 len 28672
58992024-02-03T17:25:03.889Z11/30 Write block 5 len 12288 data: 5 5 7
59002024-02-03T17:25:03.889Z12/30 Write block 185 len 8192 data: 4 5
59012024-02-03T17:25:03.889Z13/30 Write block 180 len 16384 data: 6 7 6 5
59022024-02-03T17:25:03.889Z14/30 Flush
59032024-02-03T17:25:03.894Z15/30 Flush
59042024-02-03T17:25:03.894Z16/30 Write block 576 len 36864 data: 5 6 7 8 8 7 7 6 6
59052024-02-03T17:25:03.901Z17/30 Read block 29 len 28672
59062024-02-03T17:25:03.901Z18/30 Read block 83 len 8192
59072024-02-03T17:25:03.901Z19/30 Write block 10 len 16384 data: 7 7 6 6
59082024-02-03T17:25:03.901Z20/30 Read block 245 len 20480
59092024-02-03T17:25:03.901Z21/30 Write block 282 len 28672 data: 5 5 6 5 4 6 6
59102024-02-03T17:25:03.910Z22/30 Write block 354 len 24576 data: 4 4 6 5 5 6
59112024-02-03T17:25:03.910Z23/30 Read block 28 len 16384
59122024-02-03T17:25:03.910Z24/30 Write block 357 len 28672 data: 6 6 7 7 8 8 9
59132024-02-03T17:25:03.925Z25/30 Read block 374 len 40960
59142024-02-03T17:25:03.925Z26/30 Read block 112 len 8192
59152024-02-03T17:25:03.926Z27/30 Read block 59 len 12288
59162024-02-03T17:25:03.926Z28/30 Read block 0 len 20480
59172024-02-03T17:25:03.926Z29/30 Write block 380 len 36864 data: 4 4 3 4 4 4 4 4 5
59182024-02-03T17:25:03.926Z30/30 Read block 467 len 36864
59192024-02-03T17:25:03.926Z----------------------------------------------------------------
59202024-02-03T17:25:03.927Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59212024-02-03T17:25:03.927ZStates: Active Active Active
59222024-02-03T17:25:03.927ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59232024-02-03T17:25:03.927Z 1 Acked 1000 Write 2 Done Done Done false
59242024-02-03T17:25:03.927Z 2 Acked 1001 Read 10 Done Sent Done false
59252024-02-03T17:25:03.927Z 3 Acked 1002 Write 7 Done Sent Done false
59262024-02-03T17:25:03.927Z 4 Acked 1003 Read 8 Done Sent Done false
59272024-02-03T17:25:03.928Z 5 Acked 1004 Write 10 Done Sent Done false
59282024-02-03T17:25:03.928Z 6 Acked 1005 Write 5 Done Sent Done false
59292024-02-03T17:25:03.928Z 7 Acked 1006 Write 3 Done Sent Done false
59302024-02-03T17:25:03.928Z 8 Acked 1007 Read 5 Done Sent Done false
59312024-02-03T17:25:03.928Z 9 Acked 1008 Write 3 Done Sent Done false
59322024-02-03T17:25:03.928Z 10 Acked 1009 Read 7 Done Sent Done false
59332024-02-03T17:25:03.929Z 11 Acked 1010 Write 3 Done Sent Done false
59342024-02-03T17:25:03.929Z 12 Acked 1011 Write 2 Done Sent Done false
59352024-02-03T17:25:03.929Z 13 Acked 1012 Write 4 Done Sent Done false
59362024-02-03T17:25:03.929Z 14 Acked 1013 Flush 0 Done Sent Done false
59372024-02-03T17:25:03.929Z 15 Acked 1014 Flush 0 Done Sent Done false
59382024-02-03T17:25:03.929Z 16 Acked 1015 Write 9 Done Sent Done false
59392024-02-03T17:25:03.930Z 17 Acked 1016 Read 7 Done Sent Done false
59402024-02-03T17:25:03.930Z 18 Acked 1017 Read 2 Done Sent Done false
59412024-02-03T17:25:03.930Z 19 Acked 1018 Write 4 Done Sent Done false
59422024-02-03T17:25:03.930Z 20 Acked 1019 Read 5 Done Sent Done false
59432024-02-03T17:25:03.930Z 21 Acked 1020 Write 7 Done Sent Done false
59442024-02-03T17:25:03.930Z 22 Acked 1021 Write 6 Done Sent Done false
59452024-02-03T17:25:03.930Z 23 Acked 1022 Read 4 Done Sent Done false
59462024-02-03T17:25:03.930Z 24 Acked 1023 Write 7 Done Sent Done false
59472024-02-03T17:25:03.930Z 25 Acked 1024 Read 10 Done Sent Done false
59482024-02-03T17:25:03.930Z 26 Acked 1025 Read 2 Done Sent Done false
59492024-02-03T17:25:03.930Z 27 Acked 1026 Read 3 Done Sent Done false
59502024-02-03T17:25:03.931Z 28 Acked 1027 Read 5 Done Sent Done false
59512024-02-03T17:25:03.931Z 29 Acked 1028 Write 9 Done Sent Done false
59522024-02-03T17:25:03.931Z 30 Acked 1029 Read 9 Done Sent Done false
59532024-02-03T17:25:03.931Z STATES DS:0 DS:1 DS:2 TOTAL
59542024-02-03T17:25:03.931Z New 0 0 0 0
59552024-02-03T17:25:03.931Z Sent 0 29 0 29
59562024-02-03T17:25:03.931Z Done 30 1 30 61
59572024-02-03T17:25:03.931Z Skipped 0 0 0 0
59582024-02-03T17:25:03.931Z Error 0 0 0 0
59592024-02-03T17:25:03.931ZLast Flush: 1014 0 1014
59602024-02-03T17:25:03.932ZDownstairs last five completed:
59612024-02-03T17:25:03.932ZUpstairs last five completed: 30 29 28 27 26
59622024-02-03T17:25:03.942Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:03.077874704Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1348}
59632024-02-03T17:25:03.942ZWrote out file "/tmp/test_repair_verify.data"
59642024-02-03T17:25:03.942Z
59652024-02-03T17:25:03.942ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59662024-02-03T17:25:03.973ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59672024-02-03T17:25:03.973Z 0 000-019 54 53 54 95 94 95 T F T <---
59682024-02-03T17:25:03.979Z 1 020-039 42 42 42 76 76 76 F F F
59692024-02-03T17:25:03.980Z 2 040-059 54 52 54 95 92 95 F F F <---
59702024-02-03T17:25:03.980Z 3 060-079 52 52 52 92 92 92 F F F
59712024-02-03T17:25:03.980Z 4 080-099 46 46 46 83 83 83 F F F
59722024-02-03T17:25:03.980Z 5 100-119 50 50 50 89 89 89 F F F
59732024-02-03T17:25:03.980Z 6 120-139 54 48 54 95 86 95 F F F <---
59742024-02-03T17:25:03.980Z 7 140-159 46 46 46 84 84 84 F F F
59752024-02-03T17:25:03.980Z 8 160-179 44 44 44 80 80 80 F F F
59762024-02-03T17:25:03.980Z 9 180-199 54 53 54 95 94 95 F F F <---
59772024-02-03T17:25:03.980Z 10 200-219 50 50 50 89 89 89 F F F
59782024-02-03T17:25:03.980Z 11 220-239 54 52 54 95 91 95 F F F <---
59792024-02-03T17:25:03.980Z 12 240-259 47 47 47 85 85 85 F F F
59802024-02-03T17:25:03.980Z 13 260-279 54 52 54 95 91 95 F F F <---
59812024-02-03T17:25:03.980Z 14 280-299 54 51 54 95 90 95 T T T <---
59822024-02-03T17:25:03.980Z 15 300-319 54 52 54 95 91 95 F F F <---
59832024-02-03T17:25:03.980Z 16 320-339 46 46 46 84 84 84 F F F
59842024-02-03T17:25:03.980Z 17 340-359 50 50 50 89 89 89 T F T <---
59852024-02-03T17:25:03.980Z 18 360-379 52 52 52 91 91 91 T F T <---
59862024-02-03T17:25:03.980Z 19 380-399 52 52 52 93 93 93 T F T <---
59872024-02-03T17:25:03.980Z 20 400-419 50 50 50 89 89 89 F F F
59882024-02-03T17:25:03.980Z 21 420-439 48 48 48 86 86 86 F F F
59892024-02-03T17:25:03.980Z 22 440-459 45 45 45 81 81 81 F F F
59902024-02-03T17:25:03.981Z 23 460-479 50 50 50 89 89 89 F F F
59912024-02-03T17:25:03.981Z 24 480-499 46 46 46 84 84 84 F F F
59922024-02-03T17:25:03.981Z 25 500-519 53 53 53 94 94 94 F F F
59932024-02-03T17:25:03.981Z 26 520-539 51 51 51 90 90 90 F F F
59942024-02-03T17:25:03.981Z 27 540-559 48 48 48 87 87 87 F F F
59952024-02-03T17:25:03.981Z 28 560-579 46 46 46 84 84 84 T F T <---
59962024-02-03T17:25:03.981Z 29 580-599 50 50 50 89 89 89 T F T <---
59972024-02-03T17:25:03.981ZMax gen: 54, Max flush: 95
59982024-02-03T17:25:03.981ZError: Difference in extent metadata found!
59992024-02-03T17:25:03.981ZA Difference in extent metadata is expected here
60002024-02-03T17:25:03.981ZOn loop 27
60012024-02-03T17:25:03.981Z
60022024-02-03T17:25:03.982ZVerifying data now
60032024-02-03T17:25:09.128ZNo change after verify
60042024-02-03T17:25:09.128ZLoop: 27 Downstairs dump after verify (and repair):
60052024-02-03T17:25:09.152ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60062024-02-03T17:25:09.152Z 0 000-019 55 55 55 96 96 96 F F F
60072024-02-03T17:25:09.153Z 1 020-039 42 42 42 76 76 76 F F F
60082024-02-03T17:25:09.153Z 2 040-059 54 54 54 95 95 95 F F F
60092024-02-03T17:25:09.153Z 3 060-079 52 52 52 92 92 92 F F F
60102024-02-03T17:25:09.153Z 4 080-099 46 46 46 83 83 83 F F F
60112024-02-03T17:25:09.153Z 5 100-119 50 50 50 89 89 89 F F F
60122024-02-03T17:25:09.153Z 6 120-139 54 54 54 95 95 95 F F F
60132024-02-03T17:25:09.153Z 7 140-159 46 46 46 84 84 84 F F F
60142024-02-03T17:25:09.156Z 8 160-179 44 44 44 80 80 80 F F F
60152024-02-03T17:25:09.156Z 9 180-199 54 54 54 95 95 95 F F F
60162024-02-03T17:25:09.156Z 10 200-219 50 50 50 89 89 89 F F F
60172024-02-03T17:25:09.156Z 11 220-239 54 54 54 95 95 95 F F F
60182024-02-03T17:25:09.156Z 12 240-259 47 47 47 85 85 85 F F F
60192024-02-03T17:25:09.156Z 13 260-279 54 54 54 95 95 95 F F F
60202024-02-03T17:25:09.156Z 14 280-299 55 55 55 96 96 96 F F F
60212024-02-03T17:25:09.157Z 15 300-319 54 54 54 95 95 95 F F F
60222024-02-03T17:25:09.157Z 16 320-339 46 46 46 84 84 84 F F F
60232024-02-03T17:25:09.157Z 17 340-359 55 55 55 96 96 96 F F F
60242024-02-03T17:25:09.157Z 18 360-379 55 55 55 96 96 96 F F F
60252024-02-03T17:25:09.157Z 19 380-399 55 55 55 96 96 96 F F F
60262024-02-03T17:25:09.157Z 20 400-419 50 50 50 89 89 89 F F F
60272024-02-03T17:25:09.157Z 21 420-439 48 48 48 86 86 86 F F F
60282024-02-03T17:25:09.157Z 22 440-459 45 45 45 81 81 81 F F F
60292024-02-03T17:25:09.157Z 23 460-479 50 50 50 89 89 89 F F F
60302024-02-03T17:25:09.157Z 24 480-499 46 46 46 84 84 84 F F F
60312024-02-03T17:25:09.157Z 25 500-519 53 53 53 94 94 94 F F F
60322024-02-03T17:25:09.157Z 26 520-539 51 51 51 90 90 90 F F F
60332024-02-03T17:25:09.157Z 27 540-559 48 48 48 87 87 87 F F F
60342024-02-03T17:25:09.157Z 28 560-579 55 55 55 96 96 96 F F F
60352024-02-03T17:25:09.157Z 29 580-599 55 55 55 96 96 96 F F F
60362024-02-03T17:25:09.157ZMax gen: 55, Max flush: 96
60372024-02-03T17:25:09.160Z
60382024-02-03T17:25:09.161ZBegin loop 28
60392024-02-03T17:25:09.161ZDownstairs to restart: 2
60402024-02-03T17:25:09.190Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.351426105Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356}
60412024-02-03T17:25:09.194Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.3518976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356}
60422024-02-03T17:25:09.194Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
60432024-02-03T17:25:09.194Z,"time":"2024-02-03T17:25:08.352023271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356}
60442024-02-03T17:25:09.194Z{"msg":"Crucible 3de657f2-d70e-4727-af0f-4f7c413d971b has session id: fb04fb98-bd55-4ebe-81b8-9dc0279fcd93","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.35225243Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60452024-02-03T17:25:09.194Z{"msg":"Upstairs opts: Upstairs UUID: 3de657f2-d70e-4727-af0f-4f7c413d971b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.3523045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60462024-02-03T17:25:09.194Z{"msg":"Crucible stats registered with UUID: 3de657f2-d70e-4727-af0f-4f7c413d971b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.352354509Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60472024-02-03T17:25:09.194Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.352379533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356}
60482024-02-03T17:25:09.194Z{"msg":"3de657f2-d70e-4727-af0f-4f7c413d971b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.352738169Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60492024-02-03T17:25:09.194Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.352811912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60502024-02-03T17:25:09.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.352842375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60512024-02-03T17:25:09.194Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.352914353Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60522024-02-03T17:25:09.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353000387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60532024-02-03T17:25:09.194Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353076476Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60542024-02-03T17:25:09.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353228228Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60552024-02-03T17:25:09.194Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353278232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60562024-02-03T17:25:09.194Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353320264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60572024-02-03T17:25:09.194Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353359964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60582024-02-03T17:25:09.194Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353443039Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"io task","client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60592024-02-03T17:25:09.194Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353522465Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"io task","client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60602024-02-03T17:25:09.194Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.353615171Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"io task","client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60612024-02-03T17:25:09.194Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.354381519Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60622024-02-03T17:25:09.194Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.354443885Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60632024-02-03T17:25:09.195Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.354578939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60642024-02-03T17:25:09.195Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.354641689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60652024-02-03T17:25:09.200Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.355629118Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60662024-02-03T17:25:09.200Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.355796672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60672024-02-03T17:25:09.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.35600078Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60682024-02-03T17:25:09.201Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.356159665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60692024-02-03T17:25:09.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.356868063Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60702024-02-03T17:25:09.201Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.356999695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60712024-02-03T17:25:09.201Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.357027076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60722024-02-03T17:25:09.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.357048557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60732024-02-03T17:25:09.201Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.357071134Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60742024-02-03T17:25:09.201Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.357093227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60752024-02-03T17:25:09.201Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.357246311Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60762024-02-03T17:25:09.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.3580024Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60772024-02-03T17:25:09.201Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.35805642Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60782024-02-03T17:25:09.201Z{"msg":"[0]R flush_numbers[0..12]: [96, 76, 95, 92, 83, 89, 95, 84, 80, 95, 89, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358083693Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60792024-02-03T17:25:09.201Z{"msg":"[0]R generation[0..12]: [55, 42, 54, 52, 46, 50, 54, 46, 44, 54, 50, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.35810709Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60802024-02-03T17:25:09.201Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358145217Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60812024-02-03T17:25:09.201Z{"msg":"[1]R flush_numbers[0..12]: [96, 76, 95, 92, 83, 89, 95, 84, 80, 95, 89, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358189498Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60822024-02-03T17:25:09.201Z{"msg":"[1]R generation[0..12]: [55, 42, 54, 52, 46, 50, 54, 46, 44, 54, 50, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358214939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60832024-02-03T17:25:09.201Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358239373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60842024-02-03T17:25:09.201Z{"msg":"[2]R flush_numbers[0..12]: [96, 76, 95, 92, 83, 89, 95, 84, 80, 95, 89, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358282023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60852024-02-03T17:25:09.201Z{"msg":"[2]R generation[0..12]: [55, 42, 54, 52, 46, 50, 54, 46, 44, 54, 50, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.35830659Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60862024-02-03T17:25:09.201Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358329848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60872024-02-03T17:25:09.201Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358351558Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60882024-02-03T17:25:09.201Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358373807Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60892024-02-03T17:25:09.201Z{"msg":"Next flush: 97","v":0,"Wait for a query_work_queue command to finish before sending IO
60902024-02-03T17:25:09.202Zname":"crucible","level":30,"time":"2024-02-03T17:25:08.35842043Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60912024-02-03T17:25:09.202Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358444327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"":"downstairs","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60922024-02-03T17:25:09.202Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.35846635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60932024-02-03T17:25:09.202Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358506153Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"0","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60942024-02-03T17:25:09.202Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358530325Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"1","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60952024-02-03T17:25:09.202Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358551817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"client":"2","":"downstairs","session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60962024-02-03T17:25:09.202Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358574834Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60972024-02-03T17:25:09.202Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.35859613Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60982024-02-03T17:25:09.202Z{"msg":"3de657f2-d70e-4727-af0f-4f7c413d971b is now active with session: fb04fb98-bd55-4ebe-81b8-9dc0279fcd93","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358617431Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
60992024-02-03T17:25:09.202Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358638375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356,"session_id":"fb04fb98-bd55-4ebe-81b8-9dc0279fcd93"}
61002024-02-03T17:25:09.202Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:08.358658633Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356}
61012024-02-03T17:25:09.202ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61022024-02-03T17:25:09.202ZLoading write count information from file "/tmp/test_repair_verify.data"
61032024-02-03T17:25:09.202ZRun Repair workload
61042024-02-03T17:25:09.202Z01/30 Write block 172 len 24576 data: 4 4 3 3 3 4
61052024-02-03T17:25:09.205Z02/30 Read block 344 len 32768
61062024-02-03T17:25:09.206Z03/30 Flush
61072024-02-03T17:25:09.209Z04/30 Write block 486 len 8192 data: 5 7
61082024-02-03T17:25:09.209Z05/30 Read block 452 len 32768
61092024-02-03T17:25:09.212Z06/30 Write block 23 len 24576 data: 5 5 6 4 4 4
61102024-02-03T17:25:09.212Z07/30 Write block 195 len 4096 data: 4
61112024-02-03T17:25:09.212Z08/30 Read block 238 len 36864
61122024-02-03T17:25:09.215Z09/30 Write block 126 len 20480 data: 8 7 7 9 9
61132024-02-03T17:25:09.215Z10/30 Write block 214 len 4096 data: 5
61142024-02-03T17:25:09.215Z11/30 Flush
61152024-02-03T17:25:09.218Z12/30 Write block 113 len 36864 data: 8 9 7 5 7 7 6 5 6
61162024-02-03T17:25:09.221Z13/30 Write block 481 len 24576 data: 5 5 4 3 4 6
61172024-02-03T17:25:09.222Z14/30 Write block 319 len 16384 data: 3 3 3 5
61182024-02-03T17:25:09.222Z15/30 Read block 131 len 8192
61192024-02-03T17:25:09.225Z16/30 Write block 507 len 36864 data: 10 10 10 7 5 4 3 3 3
61202024-02-03T17:25:09.225Z17/30 Read block 479 len 20480
61212024-02-03T17:25:09.230Z18/30 Read block 79 len 24576
61222024-02-03T17:25:09.231Z19/30 Read block 547 len 28672
61232024-02-03T17:25:09.234Z20/30 Write block 185 len 12288 data: 5 6 7
61242024-02-03T17:25:09.234Z21/30 Write block 396 len 4096 data: 7
61252024-02-03T17:25:09.234Z22/30 Write block 326 len 28672 data: 6 5 6 4 5 5 5
61262024-02-03T17:25:09.234Z23/30 Read block 6 len 28672
61272024-02-03T17:25:09.237Z24/30 Read block 295 len 16384
61282024-02-03T17:25:09.237Z25/30 Read block 46 len 20480
61292024-02-03T17:25:09.237Z26/30 Read block 36 len 36864
61302024-02-03T17:25:09.240Z27/30 Read block 57 len 20480
61312024-02-03T17:25:09.240Z28/30 Write block 348 len 8192 data: 4 4
61322024-02-03T17:25:09.240Z29/30 Read block 356 len 8192
61332024-02-03T17:25:09.244Z30/30 Read block 104 len 4096
61342024-02-03T17:25:09.244Z----------------------------------------------------------------
61352024-02-03T17:25:09.244Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:19
61362024-02-03T17:25:09.244ZStates: Active Active Active
61372024-02-03T17:25:09.244ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61382024-02-03T17:25:09.244Z 12 Acked 1011 Write 9 Done Done Done false
61392024-02-03T17:25:09.244Z 13 Acked 1012 Write 6 Done Done Done false
61402024-02-03T17:25:09.244Z 14 Acked 1013 Write 4 Done Done Done false
61412024-02-03T17:25:09.244Z 15 Acked 1014 Read 2 Done Done Done false
61422024-02-03T17:25:09.244Z 16 Acked 1015 Write 9 Done Done Sent false
61432024-02-03T17:25:09.244Z 17 Acked 1016 Read 5 Done Done Sent false
61442024-02-03T17:25:09.244Z 18 Acked 1017 Read 6 Done Done Sent false
61452024-02-03T17:25:09.244Z 19 Acked 1018 Read 7 Done Done Sent false
61462024-02-03T17:25:09.244Z 20 Acked 1019 Write 3 Done Done Sent false
61472024-02-03T17:25:09.244Z 21 Acked 1020 Write 1 Done Done Sent false
61482024-02-03T17:25:09.244Z 22 Acked 1021 Write 7 Done Done Sent false
61492024-02-03T17:25:09.245Z 23 Acked 1022 Read 7 Done Done Sent false
61502024-02-03T17:25:09.245Z 24 Acked 1023 Read 4 Done Done Sent false
61512024-02-03T17:25:09.245Z 25 Acked 1024 Read 5 Done Done Sent false
61522024-02-03T17:25:09.245Z 26 Acked 1025 Read 9 Done Done Sent false
61532024-02-03T17:25:09.245Z 27 Acked 1026 Read 5 Done Done Sent false
61542024-02-03T17:25:09.245Z 28 Acked 1027 Write 2 Done Done Sent false
61552024-02-03T17:25:09.245Z 29 Acked 1028 Read 2 Done Done Sent false
61562024-02-03T17:25:09.245Z 30 Acked 1029 Read 1 Done Sent Sent false
61572024-02-03T17:25:09.245Z STATES DS:0 DS:1 DS:2 TOTAL
61582024-02-03T17:25:09.245Z New 0 0 0 0
61592024-02-03T17:25:09.245Z Sent 0 1 15 16
61602024-02-03T17:25:09.245Z Done 19 18 4 41
61612024-02-03T17:25:09.245Z Skipped 0 0 0 0
61622024-02-03T17:25:09.245Z Error 0 0 0 0
61632024-02-03T17:25:09.245ZLast Flush: 1010 1010 1010
61642024-02-03T17:25:09.245ZDownstairs last five completed: 1010 1009 1008 1007 1006
61652024-02-03T17:25:09.245ZUpstairs last five completed: 30 29 28 27 26
61662024-02-03T17:25:09.245Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:08.404890289Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1356}
61672024-02-03T17:25:09.245ZWrote out file "/tmp/test_repair_verify.data"
61682024-02-03T17:25:09.250Z
61692024-02-03T17:25:09.258ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61702024-02-03T17:25:09.288ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61712024-02-03T17:25:09.288Z 0 000-019 55 55 55 96 96 96 F F F
61722024-02-03T17:25:09.289Z 1 020-039 56 56 56 98 98 98 F F F
61732024-02-03T17:25:09.289Z 2 040-059 54 54 54 95 95 95 F F F
61742024-02-03T17:25:09.289Z 3 060-079 52 52 52 92 92 92 F F F
61752024-02-03T17:25:09.289Z 4 080-099 46 46 46 83 83 83 F F F
61762024-02-03T17:25:09.292Z 5 100-119 50 50 50 89 89 89 T T T
61772024-02-03T17:25:09.292Z 6 120-139 56 56 56 98 98 98 T T T
61782024-02-03T17:25:09.292Z 7 140-159 46 46 46 84 84 84 F F F
61792024-02-03T17:25:09.292Z 8 160-179 56 56 56 97 97 97 F F F
61802024-02-03T17:25:09.292Z 9 180-199 56 56 56 98 98 98 T T F <---
61812024-02-03T17:25:09.292Z 10 200-219 56 56 56 98 98 98 F F F
61822024-02-03T17:25:09.292Z 11 220-239 54 54 54 95 95 95 F F F
61832024-02-03T17:25:09.292Z 12 240-259 47 47 47 85 85 85 F F F
61842024-02-03T17:25:09.292Z 13 260-279 54 54 54 95 95 95 F F F
61852024-02-03T17:25:09.292Z 14 280-299 55 55 55 96 96 96 F F F
61862024-02-03T17:25:09.293Z 15 300-319 54 54 54 95 95 95 T T T
61872024-02-03T17:25:09.293Z 16 320-339 46 46 46 84 84 84 T T T
61882024-02-03T17:25:09.293Z 17 340-359 55 55 55 96 96 96 T T F <---
61892024-02-03T17:25:09.293Z 18 360-379 55 55 55 96 96 96 F F F
61902024-02-03T17:25:09.293Z 19 380-399 55 55 55 96 96 96 T T F <---
61912024-02-03T17:25:09.293Z 20 400-419 50 50 50 89 89 89 F F F
61922024-02-03T17:25:09.293Z 21 420-439 48 48 48 86 86 86 F F F
61932024-02-03T17:25:09.293Z 22 440-459 45 45 45 81 81 81 F F F
61942024-02-03T17:25:09.293Z 23 460-479 50 50 50 89 89 89 F F F
61952024-02-03T17:25:09.293Z 24 480-499 56 56 56 98 98 98 T T T
61962024-02-03T17:25:09.293Z 25 500-519 53 53 53 94 94 94 T T F <---
61972024-02-03T17:25:09.293Z 26 520-539 51 51 51 90 90 90 F F F
61982024-02-03T17:25:09.293Z 27 540-559 48 48 48 87 87 87 F F F
61992024-02-03T17:25:09.293Z 28 560-579 55 55 55 96 96 96 F F F
62002024-02-03T17:25:09.293Z 29 580-599 55 55 55 96 96 96 F F F
62012024-02-03T17:25:09.293ZMax gen: 56, Max flush: 98
62022024-02-03T17:25:09.293ZError: Difference in extent metadata found!
62032024-02-03T17:25:09.296ZA Difference in extent metadata is expected here
62042024-02-03T17:25:09.296ZOn loop 28
62052024-02-03T17:25:09.297Z
62062024-02-03T17:25:09.300ZVerifying data now
62072024-02-03T17:25:15.161ZNo change after verify
62082024-02-03T17:25:15.161ZLoop: 28 Downstairs dump after verify (and repair):
62092024-02-03T17:25:15.194ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62102024-02-03T17:25:15.194Z 0 000-019 55 55 55 96 96 96 F F F
62112024-02-03T17:25:15.198Z 1 020-039 56 56 56 98 98 98 F F F
62122024-02-03T17:25:15.198Z 2 040-059 54 54 54 95 95 95 F F F
62132024-02-03T17:25:15.198Z 3 060-079 52 52 52 92 92 92 F F F
62142024-02-03T17:25:15.198Z 4 080-099 46 46 46 83 83 83 F F F
62152024-02-03T17:25:15.198Z 5 100-119 57 57 57 99 99 99 F F F
62162024-02-03T17:25:15.198Z 6 120-139 57 57 57 99 99 99 F F F
62172024-02-03T17:25:15.198Z 7 140-159 46 46 46 84 84 84 F F F
62182024-02-03T17:25:15.198Z 8 160-179 56 56 56 97 97 97 F F F
62192024-02-03T17:25:15.198Z 9 180-199 57 57 57 99 99 99 F F F
62202024-02-03T17:25:15.198Z 10 200-219 56 56 56 98 98 98 F F F
62212024-02-03T17:25:15.198Z 11 220-239 54 54 54 95 95 95 F F F
62222024-02-03T17:25:15.198Z 12 240-259 47 47 47 85 85 85 F F F
62232024-02-03T17:25:15.198Z 13 260-279 54 54 54 95 95 95 F F F
62242024-02-03T17:25:15.198Z 14 280-299 55 55 55 96 96 96 F F F
62252024-02-03T17:25:15.198Z 15 300-319 57 57 57 99 99 99 F F F
62262024-02-03T17:25:15.198Z 16 320-339 57 57 57 99 99 99 F F F
62272024-02-03T17:25:15.198Z 17 340-359 57 57 57 99 99 99 F F F
62282024-02-03T17:25:15.198Z 18 360-379 55 55 55 96 96 96 F F F
62292024-02-03T17:25:15.198Z 19 380-399 57 57 57 99 99 99 F F F
62302024-02-03T17:25:15.198Z 20 400-419 50 50 50 89 89 89 F F F
62312024-02-03T17:25:15.199Z 21 420-439 48 48 48 86 86 86 F F F
62322024-02-03T17:25:15.199Z 22 440-459 45 45 45 81 81 81 F F F
62332024-02-03T17:25:15.199Z 23 460-479 50 50 50 89 89 89 F F F
62342024-02-03T17:25:15.199Z 24 480-499 57 57 57 99 99 99 F F F
62352024-02-03T17:25:15.199Z 25 500-519 57 57 57 99 99 99 F F F
62362024-02-03T17:25:15.199Z 26 520-539 51 51 51 90 90 90 F F F
62372024-02-03T17:25:15.199Z 27 540-559 48 48 48 87 87 87 F F F
62382024-02-03T17:25:15.199Z 28 560-579 55 55 55 96 96 96 F F F
62392024-02-03T17:25:15.199Z 29 580-599 55 55 55 96 96 96 F F F
62402024-02-03T17:25:15.199ZMax gen: 57, Max flush: 99
62412024-02-03T17:25:15.199Z
62422024-02-03T17:25:15.199ZBegin loop 29
62432024-02-03T17:25:15.199ZDownstairs to restart: 1
62442024-02-03T17:25:15.233Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.386988129Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364}
62452024-02-03T17:25:15.233Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.387494105Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364}
62462024-02-03T17:25:15.233Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.387526702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364}
62472024-02-03T17:25:15.246Z{"msg":"Crucible 599f4d08-f923-4494-8811-e931732d633d has session id: 33221dcf-7d85-49e2-b449-c9e94e6b4a75","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.387563941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62482024-02-03T17:25:15.246Z{"Crucible runtime is spawned
62492024-02-03T17:25:15.246Zmsg":"Upstairs opts: Upstairs UUID: 599f4d08-f923-4494-8811-e931732d633d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.387668011Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62502024-02-03T17:25:15.246Z{"msg":"Crucible stats registered with UUID: 599f4d08-f923-4494-8811-e931732d633d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.387819889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62512024-02-03T17:25:15.246Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.387925854Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364}
62522024-02-03T17:25:15.246Z{"msg":"599f4d08-f923-4494-8811-e931732d633d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388111335Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62532024-02-03T17:25:15.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.38818188Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62542024-02-03T17:25:15.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.38821083Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62552024-02-03T17:25:15.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388244579Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62562024-02-03T17:25:15.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388283444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62572024-02-03T17:25:15.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388379162Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62582024-02-03T17:25:15.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388439842Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62592024-02-03T17:25:15.247Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388486156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62602024-02-03T17:25:15.247Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388529167Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62612024-02-03T17:25:15.247Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.38862746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62622024-02-03T17:25:15.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388817182Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"io task","client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62632024-02-03T17:25:15.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388901592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"io task","client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62642024-02-03T17:25:15.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.388991596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"io task","client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62652024-02-03T17:25:15.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.390317364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62662024-02-03T17:25:15.247Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.390423304Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62672024-02-03T17:25:15.248Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.390515131Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62682024-02-03T17:25:15.248Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.390563114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62692024-02-03T17:25:15.248Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.391213219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62702024-02-03T17:25:15.248Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.391280902Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62712024-02-03T17:25:15.248Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.391430547Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62722024-02-03T17:25:15.248Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.391609753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62732024-02-03T17:25:15.248Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.392311292Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62742024-02-03T17:25:15.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.392633628Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62752024-02-03T17:25:15.248Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.392701696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62762024-02-03T17:25:15.248Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.392747953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62772024-02-03T17:25:15.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.392801348Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62782024-02-03T17:25:15.248Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.392845069Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62792024-02-03T17:25:15.248Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.392898164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62802024-02-03T17:25:15.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393228102Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62812024-02-03T17:25:15.249Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393306729Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62822024-02-03T17:25:15.249Z{"msg":"[0]R flush_numbers[0..12]: [96, 98, 95, 92, 83, 99, 99, 84, 97, 99, 98, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393372938Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62832024-02-03T17:25:15.249Z{"msg":"[0]R generation[0..12]: [55, 56, 54, 52, 46, 57, 57, 46, 56, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393401334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62842024-02-03T17:25:15.249Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393445476Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62852024-02-03T17:25:15.249Z{"msg":"[1]R flush_numbers[0..12]: [96, 98, 95, 92, 83, 99, 99, 84, 97, 99, 98, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393470918Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62862024-02-03T17:25:15.249Z{"msg":"[1]R generation[0..12]: [55, 56, 54, 52, 46, 57, 57, 46, 56, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.39351285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
62872024-02-03T17:25:15.249Z,"pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62882024-02-03T17:25:15.249Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393543188Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62892024-02-03T17:25:15.249Z{"msg":"[2]R flush_numbers[0..12]: [96, 98, 95, 92, 83, 99, 99, 84, 97, 99, 98, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393592871Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62902024-02-03T17:25:15.249Z{"msg":"[2]R generation[0..12]: [55, 56, 54, 52, 46, 57, 57, 46, 56, 57, 56, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393617442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62912024-02-03T17:25:15.249Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393641173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62922024-02-03T17:25:15.249Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393681841Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62932024-02-03T17:25:15.249Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393779043Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62942024-02-03T17:25:15.249Z{"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393820181Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62952024-02-03T17:25:15.250Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393869536Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"":"downstairs","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62962024-02-03T17:25:15.250Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393893812Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62972024-02-03T17:25:15.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.39393317Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"0","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62982024-02-03T17:25:15.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393956845Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"1","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
62992024-02-03T17:25:15.250Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.393998212Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"client":"2","":"downstairs","session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
63002024-02-03T17:25:15.250Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.394021649Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
63012024-02-03T17:25:15.250Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.394061483Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
63022024-02-03T17:25:15.250Z{"msg":"599f4d08-f923-4494-8811-e931732d633d is now active with session: 33221dcf-7d85-49e2-b449-c9e94e6b4a75","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.394085246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
63032024-02-03T17:25:15.250Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.394123504Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364,"session_id":"33221dcf-7d85-49e2-b449-c9e94e6b4a75"}
63042024-02-03T17:25:15.250Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:14.394145511Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364}
63052024-02-03T17:25:15.250ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63062024-02-03T17:25:15.250ZLoading write count information from file "/tmp/test_repair_verify.data"
63072024-02-03T17:25:15.250ZRun Repair workload
63082024-02-03T17:25:15.250Z01/30 Write block 334 len 16384 data: 6 4 3 4
63092024-02-03T17:25:15.250Z02/30 Write block 231 len 36864 data: 5 5 5 6 5 6 6 5 7
63102024-02-03T17:25:15.250Z03/30 Flush
63112024-02-03T17:25:15.255Z04/30 Write block 270 len 8192 data: 9 10
63122024-02-03T17:25:15.255Z05/30 Write block 3 len 32768 data: 4 6 6 6 8 8 7 8
63132024-02-03T17:25:15.255Z06/30 Read block 574 len 24576
63142024-02-03T17:25:15.255Z07/30 Write block 496 len 16384 data: 4 5 5 5
63152024-02-03T17:25:15.255Z08/30 Read block 593 len 4096
63162024-02-03T17:25:15.255Z09/30 Flush
63172024-02-03T17:25:15.255Z10/30 Write block 264 len 12288 data: 5 4 5
63182024-02-03T17:25:15.256Z11/30 Read block 87 len 36864
63192024-02-03T17:25:15.256Z12/30 Flush
63202024-02-03T17:25:15.259Z13/30 Write block 294 len 20480 data: 5 3 3 4 3
63212024-02-03T17:25:15.259Z14/30 Flush
63222024-02-03T17:25:15.263Z15/30 Write block 293 len 32768 data: 7 6 4 4 5 4 2 3
63232024-02-03T17:25:15.263Z16/30 Flush
63242024-02-03T17:25:15.266Z17/30 Flush
63252024-02-03T17:25:15.266Z18/30 Write block 181 len 8192 data: 8 7
63262024-02-03T17:25:15.266Z19/30 Read block 216 len 24576
63272024-02-03T17:25:15.269Z20/30 Read block 60 len 40960
63282024-02-03T17:25:15.269Z21/30 Read block 160 len 28672
63292024-02-03T17:25:15.274Z22/30 Read block 490 len 8192
63302024-02-03T17:25:15.274Z23/30 Read block 108 len 36864
63312024-02-03T17:25:15.274Z24/30 Read block 54 len 4096
63322024-02-03T17:25:15.280Z25/30 Write block 177 len 8192 data: 5 4
63332024-02-03T17:25:15.280Z26/30 Read block 334 len 12288
63342024-02-03T17:25:15.280Z27/30 Write block 239 len 32768 data: 8 5 5 3 2 2 3 5
63352024-02-03T17:25:15.280Z28/30 Read block 126 len 24576
63362024-02-03T17:25:15.280Z29/30 Read block 466 len 16384
63372024-02-03T17:25:15.284Z30/30 Write block 134 len 12288 data: 6 6 9
63382024-02-03T17:25:15.284Z----------------------------------------------------------------
63392024-02-03T17:25:15.284Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63402024-02-03T17:25:15.284ZStates: Active Active Active
63412024-02-03T17:25:15.284ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63422024-02-03T17:25:15.285Z 1 Acked 1000 Write 4 Done Done Done false
63432024-02-03T17:25:15.285Z 2 Acked 1001 Write 9 Done Sent Done false
63442024-02-03T17:25:15.285Z 3 Acked 1002 Flush 0 Done Sent Done false
63452024-02-03T17:25:15.285Z 4 Acked 1003 Write 2 Done Sent Done false
63462024-02-03T17:25:15.285Z 5 Acked 1004 Write 8 Done Sent Done false
63472024-02-03T17:25:15.285Z 6 Acked 1005 Read 6 Done Sent Done false
63482024-02-03T17:25:15.285Z 7 Acked 1006 Write 4 Done Sent Done false
63492024-02-03T17:25:15.285Z 8 Acked 1007 Read 1 Done Sent Done false
63502024-02-03T17:25:15.285Z 9 Acked 1008 Flush 0 Done Sent Done false
63512024-02-03T17:25:15.285Z 10 Acked 1009 Write 3 Done Sent Done false
63522024-02-03T17:25:15.285Z 11 Acked 1010 Read 9 Done Sent Done false
63532024-02-03T17:25:15.285Z 12 Acked 1011 Flush 0 Done Sent Done false
63542024-02-03T17:25:15.286Z 13 Acked 1012 Write 5 Done Sent Done false
63552024-02-03T17:25:15.286Z 14 Acked 1013 Flush 0 Done Sent Done false
63562024-02-03T17:25:15.286Z 15 Acked 1014 Write 8 Done Sent Done false
63572024-02-03T17:25:15.286Z 16 Acked 1015 Flush 0 Done Sent Done false
63582024-02-03T17:25:15.286Z 17 Acked 1016 Flush 0 Done Sent Done false
63592024-02-03T17:25:15.286Z 18 Acked 1017 Write 2 Done Sent Done false
63602024-02-03T17:25:15.286Z 19 Acked 1018 Read 6 Done Sent Done false
63612024-02-03T17:25:15.286Z 20 Acked 1019 Read 10 Done Sent Done false
63622024-02-03T17:25:15.286Z 21 Acked 1020 Read 7 Done Sent Done false
63632024-02-03T17:25:15.286Z 22 Acked 1021 Read 2 Done Sent Done false
63642024-02-03T17:25:15.286Z 23 Acked 1022 Read 9 Done Sent Done false
63652024-02-03T17:25:15.286Z 24 Acked 1023 Read 1 Done Sent Done false
63662024-02-03T17:25:15.286Z 25 Acked 1024 Write 2 Done Sent Done false
63672024-02-03T17:25:15.287Z 26 Acked 1025 Read 3 Done Sent Done false
63682024-02-03T17:25:15.287Z 27 Acked 1026 Write 8 Done Sent Done false
63692024-02-03T17:25:15.287Z 28 Acked 1027 Read 6 Done Sent Done false
63702024-02-03T17:25:15.287Z 29 Acked 1028 Read 4 Done Sent Done false
63712024-02-03T17:25:15.287Z 30 Acked 1029 Write 3 Sent Sent Sent false
63722024-02-03T17:25:15.287Z STATES DS:0 DS:1 DS:2 TOTAL
63732024-02-03T17:25:15.287Z New 0 0 0 0
63742024-02-03T17:25:15.287Z Sent 1 29 1 31
63752024-02-03T17:25:15.287Z Done 29 1 29 59
63762024-02-03T17:25:15.287Z Skipped 0 0 0 0
63772024-02-03T17:25:15.287Z Error 0 0 0 0
63782024-02-03T17:25:15.287ZLast Flush: 1016 0 1016
63792024-02-03T17:25:15.288ZDownstairs last five completed:
63802024-02-03T17:25:15.288ZUpstairs last five completed: 30 29 28 27 26
63812024-02-03T17:25:15.288Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:14.444003996Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1364}
63822024-02-03T17:25:15.288ZWrote out file "/tmp/test_repair_verify.data"
63832024-02-03T17:25:15.292Z
63842024-02-03T17:25:15.297ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63852024-02-03T17:25:15.324ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63862024-02-03T17:25:15.324Z 0 000-019 58 55 58 101 96 101 F F F <---
63872024-02-03T17:25:15.330Z 1 020-039 56 56 56 98 98 98 F F F
63882024-02-03T17:25:15.330Z 2 040-059 54 54 54 95 95 95 F F F
63892024-02-03T17:25:15.330Z 3 060-079 52 52 52 92 92 92 F F F
63902024-02-03T17:25:15.330Z 4 080-099 46 46 46 83 83 83 F F F
63912024-02-03T17:25:15.330Z 5 100-119 57 57 57 99 99 99 F F F
63922024-02-03T17:25:15.330Z 6 120-139 57 57 57 99 99 99 T F T <---
63932024-02-03T17:25:15.330Z 7 140-159 46 46 46 84 84 84 F F F
63942024-02-03T17:25:15.330Z 8 160-179 56 56 56 97 97 97 T F T <---
63952024-02-03T17:25:15.330Z 9 180-199 57 57 57 99 99 99 T F T <---
63962024-02-03T17:25:15.330Z 10 200-219 56 56 56 98 98 98 F F F
63972024-02-03T17:25:15.330Z 11 220-239 58 54 58 100 95 100 T F T <---
63982024-02-03T17:25:15.330Z 12 240-259 47 47 47 85 85 85 T F T <---
63992024-02-03T17:25:15.330Z 13 260-279 58 54 58 102 95 102 F F F <---
64002024-02-03T17:25:15.331Z 14 280-299 58 55 58 104 96 104 F F F <---
64012024-02-03T17:25:15.331Z 15 300-319 58 57 58 104 99 104 F F F <---
64022024-02-03T17:25:15.331Z 16 320-339 58 57 58 100 99 100 F T F <---
64032024-02-03T17:25:15.331Z 17 340-359 57 57 57 99 99 99 F F F
64042024-02-03T17:25:15.331Z 18 360-379 55 55 55 96 96 96 F F F
64052024-02-03T17:25:15.331Z 19 380-399 57 57 57 99 99 99 F F F
64062024-02-03T17:25:15.331Z 20 400-419 50 50 50 89 89 89 F F F
64072024-02-03T17:25:15.331Z 21 420-439 48 48 48 86 86 86 F F F
64082024-02-03T17:25:15.331Z 22 440-459 45 45 45 81 81 81 F F F
64092024-02-03T17:25:15.331Z 23 460-479 50 50 50 89 89 89 F F F
64102024-02-03T17:25:15.331Z 24 480-499 58 57 58 101 99 101 F F F <---
64112024-02-03T17:25:15.331Z 25 500-519 57 57 57 99 99 99 F F F
64122024-02-03T17:25:15.331Z 26 520-539 51 51 51 90 90 90 F F F
64132024-02-03T17:25:15.331Z 27 540-559 48 48 48 87 87 87 F F F
64142024-02-03T17:25:15.331Z 28 560-579 55 55 55 96 96 96 F F F
64152024-02-03T17:25:15.332Z 29 580-599 55 55 55 96 96 96 F F F
64162024-02-03T17:25:15.332ZMax gen: 58, Max flush: 104
64172024-02-03T17:25:15.332ZError: Difference in extent metadata found!
64182024-02-03T17:25:15.338ZA Difference in extent metadata is expected here
64192024-02-03T17:25:15.338ZOn loop 29
64202024-02-03T17:25:15.338Z
64212024-02-03T17:25:15.338ZVerifying data now
64222024-02-03T17:25:20.335ZNo change after verify
64232024-02-03T17:25:20.335ZLoop: 29 Downstairs dump after verify (and repair):
64242024-02-03T17:25:20.359ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64252024-02-03T17:25:20.359Z 0 000-019 58 58 58 101 101 101 F F F
64262024-02-03T17:25:20.359Z 1 020-039 56 56 56 98 98 98 F F F
64272024-02-03T17:25:20.359Z 2 040-059 54 54 54 95 95 95 F F F
64282024-02-03T17:25:20.383Z 3 060-079 52 52 52 92 92 92 F F F
64292024-02-03T17:25:20.383Z 4 080-099 46 46 46 83 83 83 F F F
64302024-02-03T17:25:20.383Z 5 100-119 57 57 57 99 99 99 F F F
64312024-02-03T17:25:20.383Z 6 120-139 59 59 59 105 105 105 F F F
64322024-02-03T17:25:20.383Z 7 140-159 46 46 46 84 84 84 F F F
64332024-02-03T17:25:20.383Z 8 160-179 59 59 59 105 105 105 F F F
64342024-02-03T17:25:20.383Z 9 180-199 59 59 59 105 105 105 F F F
64352024-02-03T17:25:20.383Z 10 200-219 56 56 56 98 98 98 F F F
64362024-02-03T17:25:20.384Z 11 220-239 59 59 59 105 105 105 F F F
64372024-02-03T17:25:20.384Z 12 240-259 59 59 59 105 105 105 F F F
64382024-02-03T17:25:20.384Z 13 260-279 58 58 58 102 102 102 F F F
64392024-02-03T17:25:20.384Z 14 280-299 58 58 58 104 104 104 F F F
64402024-02-03T17:25:20.384Z 15 300-319 58 58 58 104 104 104 F F F
64412024-02-03T17:25:20.384Z 16 320-339 58 58 58 100 100 100 F F F
64422024-02-03T17:25:20.384Z 17 340-359 57 57 57 99 99 99 F F F
64432024-02-03T17:25:20.384Z 18 360-379 55 55 55 96 96 96 F F F
64442024-02-03T17:25:20.384Z 19 380-399 57 57 57 99 99 99 F F F
64452024-02-03T17:25:20.384Z 20 400-419 50 50 50 89 89 89 F F F
64462024-02-03T17:25:20.384Z 21 420-439 48 48 48 86 86 86 F F F
64472024-02-03T17:25:20.384Z 22 440-459 45 45 45 81 81 81 F F F
64482024-02-03T17:25:20.384Z 23 460-479 50 50 50 89 89 89 F F F
64492024-02-03T17:25:20.384Z 24 480-499 58 58 58 101 101 101 F F F
64502024-02-03T17:25:20.384Z 25 500-519 57 57 57 99 99 99 F F F
64512024-02-03T17:25:20.384Z 26 520-539 51 51 51 90 90 90 F F F
64522024-02-03T17:25:20.384Z 27 540-559 48 48 48 87 87 87 F F F
64532024-02-03T17:25:20.384Z 28 560-579 55 55 55 96 96 96 F F F
64542024-02-03T17:25:20.385Z 29 580-599 55 55 55 96 96 96 F F F
64552024-02-03T17:25:20.385ZMax gen: 59, Max flush: 105
64562024-02-03T17:25:20.390Z
64572024-02-03T17:25:20.390ZBegin loop 30
64582024-02-03T17:25:20.390ZDownstairs to restart: 1
64592024-02-03T17:25:20.397Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.558940486Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372}
64602024-02-03T17:25:20.397Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.559515397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372}
64612024-02-03T17:25:20.397Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.559594506Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372}
64622024-02-03T17:25:20.403ZCrucible runtime is spawned
64632024-02-03T17:25:20.403Z{"msg":"Crucible 5ffb419f-6f70-4a1d-ae3a-410662049985 has session id: 308fb531-45ae-43dd-8e73-74b995de190c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.559688557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64642024-02-03T17:25:20.403Z{"msg":"Upstairs opts: Upstairs UUID: 5ffb419f-6f70-4a1d-ae3a-410662049985, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.559809088Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64652024-02-03T17:25:20.403Z{"msg":"Crucible stats registered with UUID: 5ffb419f-6f70-4a1d-ae3a-410662049985","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.559857218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64662024-02-03T17:25:20.403Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.5598954Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372}
64672024-02-03T17:25:20.403Z{"msg":"5ffb419f-6f70-4a1d-ae3a-410662049985 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560003726Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64682024-02-03T17:25:20.403Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560108132Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64692024-02-03T17:25:20.403Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560253632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64702024-02-03T17:25:20.403Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560376019Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64712024-02-03T17:25:20.403Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.56057554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64722024-02-03T17:25:20.403Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560672922Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64732024-02-03T17:25:20.403Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.56075041Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64742024-02-03T17:25:20.403Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560836908Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64752024-02-03T17:25:20.403Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560911418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64762024-02-03T17:25:20.403Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560954042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64772024-02-03T17:25:20.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.560992997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"io task","client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64782024-02-03T17:25:20.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.56106611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"io task","client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64792024-02-03T17:25:20.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.561108825Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"io task","client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64802024-02-03T17:25:20.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.561787342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64812024-02-03T17:25:20.404Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.561913996Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64822024-02-03T17:25:20.404Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.561975228Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64832024-02-03T17:25:20.404Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.562022455Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64842024-02-03T17:25:20.404Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.562933205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64852024-02-03T17:25:20.404Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.563142983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64862024-02-03T17:25:20.413Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.563937068Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64872024-02-03T17:25:20.413Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.564040803Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64882024-02-03T17:25:20.413Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.564119372Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64892024-02-03T17:25:20.413Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.564161132Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64902024-02-03T17:25:20.413Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.564255375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64912024-02-03T17:25:20.413Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.564300885Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64922024-02-03T17:25:20.413Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.564354199Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64932024-02-03T17:25:20.413Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.564380042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64942024-02-03T17:25:20.413Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.565018652Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64952024-02-03T17:25:20.413Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.565791927Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64962024-02-03T17:25:20.413Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.56585577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64972024-02-03T17:25:20.413Z{"msg":"[0]R flush_numbers[0..12]: [101, 98, 95, 92, 83, 99, 105, 84, 105, 105, 98, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.565884694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64982024-02-03T17:25:20.413Z{"msg":"[0]R generation[0..12]: [58, 56, 54, 52, 46, 57, 59, 46, 59, 59, 56, 59]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.565909131Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
64992024-02-03T17:25:20.413Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.56593394Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65002024-02-03T17:25:20.413Z{"msg":"[1]R flush_numbers[0..12]: [101, 98, 95, 92, 83, 99, 105, 84, 105, 105, 98, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.565976609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65012024-02-03T17:25:20.413Z{"msg":"[1]R generation[0..12]: [58, 56, 54, 52, 46, 57, 59, 46, 59, 59, 56, 59]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566002254Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65022024-02-03T17:25:20.413Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566044288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65032024-02-03T17:25:20.413Z{"msg":"[2]R flush_numbers[0..12]: [101, 98, 95, 92, 83, 99, 105, 84, 105, 105, 98, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566069756Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65042024-02-03T17:25:20.413Z{"msg":"[2]R generation[0..12]: [58, 56, 54, 52, 46, 57, 59, 46, 59, 59, 56, 59]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566092466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65052024-02-03T17:25:20.413Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566116038Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65062024-02-03T17:25:20.414Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
65072024-02-03T17:25:20.414Z,"time":"2024-02-03T17:25:19.566173384Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65082024-02-03T17:25:20.414Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566244797Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65092024-02-03T17:25:20.414Z{"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566303065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65102024-02-03T17:25:20.414Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566327666Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"":"downstairs","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65112024-02-03T17:25:20.414Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566350152Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65122024-02-03T17:25:20.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566389817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"0","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65132024-02-03T17:25:20.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566413696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"1","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65142024-02-03T17:25:20.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566479952Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"client":"2","":"downstairs","session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65152024-02-03T17:25:20.414Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566546726Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65162024-02-03T17:25:20.414Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566572031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65172024-02-03T17:25:20.414Z{"msg":"5ffb419f-6f70-4a1d-ae3a-410662049985 is now active with session: 308fb531-45ae-43dd-8e73-74b995de190c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566648003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65182024-02-03T17:25:20.414Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.56671689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372,"session_id":"308fb531-45ae-43dd-8e73-74b995de190c"}
65192024-02-03T17:25:20.414Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:19.566788273Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372}
65202024-02-03T17:25:20.414ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65212024-02-03T17:25:20.414ZLoading write count information from file "/tmp/test_repair_verify.data"
65222024-02-03T17:25:20.414ZRun Repair workload
65232024-02-03T17:25:20.414Z01/30 Write block 559 len 4096 data: 6
65242024-02-03T17:25:20.414Z02/30 Read block 544 len 28672
65252024-02-03T17:25:20.420Z03/30 Write block 187 len 32768 data: 8 6 7 6 5 3 4 3
65262024-02-03T17:25:20.420Z04/30 Write block 319 len 4096 data: 4
65272024-02-03T17:25:20.420Z05/30 Read block 38 len 28672
65282024-02-03T17:25:20.420Z06/30 Flush
65292024-02-03T17:25:20.420Z07/30 Write block 390 len 4096 data: 3
65302024-02-03T17:25:20.420Z08/30 Read block 369 len 28672
65312024-02-03T17:25:20.420Z09/30 Read block 2 len 8192
65322024-02-03T17:25:20.430Z10/30 Write block 100 len 24576 data: 4 4 5 5 6 8
65332024-02-03T17:25:20.430Z11/30 Write block 42 len 8192 data: 5 5
65342024-02-03T17:25:20.430Z12/30 Write block 90 len 36864 data: 5 4 4 5 4 5 4 3 3
65352024-02-03T17:25:20.430Z13/30 Read block 517 len 24576
65362024-02-03T17:25:20.430Z14/30 Flush
65372024-02-03T17:25:20.434Z15/30 Write block 457 len 12288 data: 6 5 5
65382024-02-03T17:25:20.434Z16/30 Write block 182 len 8192 data: 8 6
65392024-02-03T17:25:20.434Z17/30 Read block 251 len 4096
65402024-02-03T17:25:20.434Z18/30 Read block 142 len 16384
65412024-02-03T17:25:20.434Z19/30 Read block 307 len 28672
65422024-02-03T17:25:20.438Z20/30 Read block 407 len 36864
65432024-02-03T17:25:20.438Z21/30 Write block 505 len 28672 data: 10 12 11 11 11 8 6
65442024-02-03T17:25:20.442Z22/30 Read block 317 len 16384
65452024-02-03T17:25:20.442Z23/30 Read block 334 len 28672
65462024-02-03T17:25:20.447Z24/30 Read block 6 len 28672
65472024-02-03T17:25:20.447Z25/30 Write block 376 len 28672 data: 5 5 5 4 5 5 4
65482024-02-03T17:25:20.447Z26/30 Read block 406 len 8192
65492024-02-03T17:25:20.447Z27/30 Write block 225 len 16384 data: 8 6 9 8
65502024-02-03T17:25:20.447Z28/30 Write block 38 len 32768 data: 6 7 6 6 6 6 6 5
65512024-02-03T17:25:20.452Z29/30 Read block 26 len 32768
65522024-02-03T17:25:20.452Z30/30 Write block 28 len 16384 data: 5 4 2 4
65532024-02-03T17:25:20.457Z----------------------------------------------------------------
65542024-02-03T17:25:20.457Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65552024-02-03T17:25:20.457ZStates: Active Active Active
65562024-02-03T17:25:20.457ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65572024-02-03T17:25:20.457Z 1 Acked 1000 Write 1 Done Done Done false
65582024-02-03T17:25:20.457Z 2 Acked 1001 Read 7 Done Sent Done false
65592024-02-03T17:25:20.457Z 3 Acked 1002 Write 8 Done Sent Done false
65602024-02-03T17:25:20.457Z 4 Acked 1003 Write 1 Done Sent Done false
65612024-02-03T17:25:20.457Z 5 Acked 1004 Read 7 Done Sent Done false
65622024-02-03T17:25:20.458Z 6 Acked 1005 Flush 0 Done Sent Done false
65632024-02-03T17:25:20.458Z 7 Acked 1006 Write 1 Done Sent Done false
65642024-02-03T17:25:20.458Z 8 Acked 1007 Read 7 Done Sent Done false
65652024-02-03T17:25:20.458Z 9 Acked 1008 Read 2 Done Sent Done false
65662024-02-03T17:25:20.458Z 10 Acked 1009 Write 6 Done Sent Done false
65672024-02-03T17:25:20.458Z 11 Acked 1010 Write 2 Done Sent Done false
65682024-02-03T17:25:20.458Z 12 Acked 1011 Write 9 Done Sent Done false
65692024-02-03T17:25:20.458Z 13 Acked 1012 Read 6 Done Sent Done false
65702024-02-03T17:25:20.458Z 14 Acked 1013 Flush 0 Done Sent Done false
65712024-02-03T17:25:20.458Z 15 Acked 1014 Write 3 Done Sent Done false
65722024-02-03T17:25:20.458Z 16 Acked 1015 Write 2 Done Sent Done false
65732024-02-03T17:25:20.458Z 17 Acked 1016 Read 1 Done Sent Done false
65742024-02-03T17:25:20.458Z 18 Acked 1017 Read 4 Done Sent Done false
65752024-02-03T17:25:20.458Z 19 Acked 1018 Read 7 Done Sent Done false
65762024-02-03T17:25:20.458Z 20 Acked 1019 Read 9 Done Sent Done false
65772024-02-03T17:25:20.458Z 21 Acked 1020 Write 7 Done Sent Done false
65782024-02-03T17:25:20.458Z 22 Acked 1021 Read 4 Done Sent Done false
65792024-02-03T17:25:20.458Z 23 Acked 1022 Read 7 Done Sent Done false
65802024-02-03T17:25:20.458Z 24 Acked 1023 Read 7 Done Sent Done false
65812024-02-03T17:25:20.458Z 25 Acked 1024 Write 7 Done Sent Done false
65822024-02-03T17:25:20.458Z 26 Acked 1025 Read 2 Done Sent Done false
65832024-02-03T17:25:20.458Z 27 Acked 1026 Write 4 Done Sent Done false
65842024-02-03T17:25:20.458Z 28 Acked 1027 Write 8 Done Sent Done false
65852024-02-03T17:25:20.458Z 29 Acked 1028 Read 8 Done Sent Done false
65862024-02-03T17:25:20.458Z 30 Acked 1029 Write 4 Sent Sent Sent false
65872024-02-03T17:25:20.459Z STATES DS:0 DS:1 DS:2 TOTAL
65882024-02-03T17:25:20.459Z New 0 0 0 0
65892024-02-03T17:25:20.459Z Sent 1 29 1 31
65902024-02-03T17:25:20.459Z Done 29 1 29 59
65912024-02-03T17:25:20.459Z Skipped 0 0 0 0
65922024-02-03T17:25:20.459Z Error 0 0 0 0
65932024-02-03T17:25:20.459ZLast Flush: 1013 0 1013
65942024-02-03T17:25:20.459ZDownstairs last five completed:
65952024-02-03T17:25:20.459ZUpstairs last five completed: 30 29 28 27 26
65962024-02-03T17:25:20.459Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:19.613897651Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1372}
65972024-02-03T17:25:20.459ZWrote out file "/tmp/test_repair_verify.data"
65982024-02-03T17:25:20.463Z
65992024-02-03T17:25:20.469ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66002024-02-03T17:25:20.500ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66012024-02-03T17:25:20.500Z 0 000-019 58 58 58 101 101 101 F F F
66022024-02-03T17:25:20.500Z 1 020-039 56 56 56 98 98 98 T F T <---
66032024-02-03T17:25:20.500Z 2 040-059 60 54 60 107 95 107 T F T <---
66042024-02-03T17:25:20.500Z 3 060-079 52 52 52 92 92 92 F F F
66052024-02-03T17:25:20.507Z 4 080-099 60 46 60 107 83 107 F F F <---
66062024-02-03T17:25:20.507Z 5 100-119 60 57 60 107 99 107 F F F <---
66072024-02-03T17:25:20.507Z 6 120-139 59 59 59 105 105 105 F F F
66082024-02-03T17:25:20.508Z 7 140-159 46 46 46 84 84 84 F F F
66092024-02-03T17:25:20.508Z 8 160-179 59 59 59 105 105 105 F F F
66102024-02-03T17:25:20.508Z 9 180-199 60 59 60 106 105 106 T F T <---
66112024-02-03T17:25:20.508Z 10 200-219 56 56 56 98 98 98 F F F
66122024-02-03T17:25:20.508Z 11 220-239 59 59 59 105 105 105 T F T <---
66132024-02-03T17:25:20.508Z 12 240-259 59 59 59 105 105 105 F F F
66142024-02-03T17:25:20.508Z 13 260-279 58 58 58 102 102 102 F F F
66152024-02-03T17:25:20.508Z 14 280-299 58 58 58 104 104 104 F F F
66162024-02-03T17:25:20.508Z 15 300-319 60 58 60 106 104 106 F F F <---
66172024-02-03T17:25:20.508Z 16 320-339 58 58 58 100 100 100 F F F
66182024-02-03T17:25:20.508Z 17 340-359 57 57 57 99 99 99 F F F
66192024-02-03T17:25:20.509Z 18 360-379 55 55 55 96 96 96 T F T <---
66202024-02-03T17:25:20.509Z 19 380-399 60 57 60 107 99 107 T F T <---
66212024-02-03T17:25:20.509Z 20 400-419 50 50 50 89 89 89 F F F
66222024-02-03T17:25:20.509Z 21 420-439 48 48 48 86 86 86 F F F
66232024-02-03T17:25:20.509Z 22 440-459 45 45 45 81 81 81 T F T <---
66242024-02-03T17:25:20.509Z 23 460-479 50 50 50 89 89 89 F F F
66252024-02-03T17:25:20.509Z 24 480-499 58 58 58 101 101 101 F F F
66262024-02-03T17:25:20.509Z 25 500-519 57 57 57 99 99 99 T F T <---
66272024-02-03T17:25:20.509Z 26 520-539 51 51 51 90 90 90 F F F
66282024-02-03T17:25:20.509Z 27 540-559 60 48 60 106 87 106 F T F <---
66292024-02-03T17:25:20.509Z 28 560-579 55 55 55 96 96 96 F F F
66302024-02-03T17:25:20.509Z 29 580-599 55 55 55 96 96 96 F F F
66312024-02-03T17:25:20.510ZMax gen: 60, Max flush: 107
66322024-02-03T17:25:20.510ZError: Difference in extent metadata found!
66332024-02-03T17:25:20.510ZA Difference in extent metadata is expected here
66342024-02-03T17:25:20.510ZOn loop 30
66352024-02-03T17:25:20.510Z
66362024-02-03T17:25:20.516ZVerifying data now
66372024-02-03T17:25:25.595ZNo change after verify
66382024-02-03T17:25:25.595ZLoop: 30 Downstairs dump after verify (and repair):
66392024-02-03T17:25:25.621ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66402024-02-03T17:25:25.621Z 0 000-019 58 58 58 101 101 101 F F F
66412024-02-03T17:25:25.624Z 1 020-039 61 61 61 108 108 108 F F F
66422024-02-03T17:25:25.625Z 2 040-059 61 61 61 108 108 108 F F F
66432024-02-03T17:25:25.625Z 3 060-079 52 52 52 92 92 92 F F F
66442024-02-03T17:25:25.625Z 4 080-099 60 60 60 107 107 107 F F F
66452024-02-03T17:25:25.625Z 5 100-119 60 60 60 107 107 107 F F F
66462024-02-03T17:25:25.625Z 6 120-139 59 59 59 105 105 105 F F F
66472024-02-03T17:25:25.625Z 7 140-159 46 46 46 84 84 84 F F F
66482024-02-03T17:25:25.625Z 8 160-179 59 59 59 105 105 105 F F F
66492024-02-03T17:25:25.625Z 9 180-199 61 61 61 108 108 108 F F F
66502024-02-03T17:25:25.625Z 10 200-219 56 56 56 98 98 98 F F F
66512024-02-03T17:25:25.625Z 11 220-239 61 61 61 108 108 108 F F F
66522024-02-03T17:25:25.625Z 12 240-259 59 59 59 105 105 105 F F F
66532024-02-03T17:25:25.625Z 13 260-279 58 58 58 102 102 102 F F F
66542024-02-03T17:25:25.626Z 14 280-299 58 58 58 104 104 104 F F F
66552024-02-03T17:25:25.626Z 15 300-319 60 60 60 106 106 106 F F F
66562024-02-03T17:25:25.626Z 16 320-339 58 58 58 100 100 100 F F F
66572024-02-03T17:25:25.626Z 17 340-359 57 57 57 99 99 99 F F F
66582024-02-03T17:25:25.626Z 18 360-379 61 61 61 108 108 108 F F F
66592024-02-03T17:25:25.626Z 19 380-399 61 61 61 108 108 108 F F F
66602024-02-03T17:25:25.626Z 20 400-419 50 50 50 89 89 89 F F F
66612024-02-03T17:25:25.626Z 21 420-439 48 48 48 86 86 86 F F F
66622024-02-03T17:25:25.626Z 22 440-459 61 61 61 108 108 108 F F F
66632024-02-03T17:25:25.626Z 23 460-479 50 50 50 89 89 89 F F F
66642024-02-03T17:25:25.626Z 24 480-499 58 58 58 101 101 101 F F F
66652024-02-03T17:25:25.626Z 25 500-519 61 61 61 108 108 108 F F F
66662024-02-03T17:25:25.626Z 26 520-539 51 51 51 90 90 90 F F F
66672024-02-03T17:25:25.626Z 27 540-559 60 60 60 106 106 106 F F F
66682024-02-03T17:25:25.627Z 28 560-579 55 55 55 96 96 96 F F F
66692024-02-03T17:25:25.627Z 29 580-599 55 55 55 96 96 96 F F F
66702024-02-03T17:25:25.627ZMax gen: 61, Max flush: 108
66712024-02-03T17:25:25.630Z
66722024-02-03T17:25:25.630ZBegin loop 31
66732024-02-03T17:25:25.631ZDownstairs to restart: 2
66742024-02-03T17:25:25.660Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.821909455Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380}
66752024-02-03T17:25:25.660Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.822487976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380}
66762024-02-03T17:25:25.664Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.822616523Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380}
66772024-02-03T17:25:25.664Z{"msg":"Crucible 8b38c238-a708-447c-8861-4f45ef917086 has session id: b9c132fe-e292-4357-954c-4b9fae4570b3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.822658254Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66782024-02-03T17:25:25.664Z{"msg":"Upstairs opts: Upstairs UUID: 8b38c238-a708-447c-8861-4f45ef917086, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.822711731Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66792024-02-03T17:25:25.664Z{"msg":"Crucible stats registered with UUID: 8b38c238-a708-447c-8861-4f45ef917086","v":0,"name":"crucible","level":30Crucible runtime is spawned
66802024-02-03T17:25:25.664Z,"time":"2024-02-03T17:25:24.822752802Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66812024-02-03T17:25:25.665Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.822949642Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380}
66822024-02-03T17:25:25.665Z{"msg":"8b38c238-a708-447c-8861-4f45ef917086 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823148434Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66832024-02-03T17:25:25.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823206396Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66842024-02-03T17:25:25.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823233863Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66852024-02-03T17:25:25.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823256665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66862024-02-03T17:25:25.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823279356Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66872024-02-03T17:25:25.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823331186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66882024-02-03T17:25:25.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823367874Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66892024-02-03T17:25:25.665Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823608128Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66902024-02-03T17:25:25.665Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.82366578Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66912024-02-03T17:25:25.665Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823880937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66922024-02-03T17:25:25.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.823988856Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"io task","client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66932024-02-03T17:25:25.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.824037586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"io task","client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66942024-02-03T17:25:25.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.824107845Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"io task","client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66952024-02-03T17:25:25.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.825240352Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66962024-02-03T17:25:25.666Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.82530903Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66972024-02-03T17:25:25.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.825335678Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66982024-02-03T17:25:25.666Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.825359282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
66992024-02-03T17:25:25.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.826019193Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67002024-02-03T17:25:25.666Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.826099672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67012024-02-03T17:25:25.671Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.826254773Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67022024-02-03T17:25:25.671Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.826524215Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67032024-02-03T17:25:25.671Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827058792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67042024-02-03T17:25:25.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.82719313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67052024-02-03T17:25:25.671Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827294667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67062024-02-03T17:25:25.671Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827466098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67072024-02-03T17:25:25.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827494623Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67082024-02-03T17:25:25.672Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827538226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67092024-02-03T17:25:25.672Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827562717Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67102024-02-03T17:25:25.672Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827787907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67112024-02-03T17:25:25.672Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827870389Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67122024-02-03T17:25:25.672Z{"msg":"[0]R flush_numbers[0..12]: [101, 108, 108, 92, 107, 107, 105, 84, 105, 108, 98, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827915944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67132024-02-03T17:25:25.672Z{"msg":"[0]R generation[0..12]: [58, 61, 61, 52, 60, 60, 59, 46, 59, 61, 56, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.827987905Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67142024-02-03T17:25:25.672Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828061159Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67152024-02-03T17:25:25.672Z{"msg":"[1]R flush_numbers[0..12]: [101, 108, 108, 92, 107, 107, 105, 84, 105, 108, 98, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828135276Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67162024-02-03T17:25:25.672Z{"msg":"[1]R generation[0..12]: [58, 61, 61, 52, 60, 60, 59, 46, 59, 61, 56, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828206018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
67172024-02-03T17:25:25.672Z,"":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67182024-02-03T17:25:25.672Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828289206Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67192024-02-03T17:25:25.672Z{"msg":"[2]R flush_numbers[0..12]: [101, 108, 108, 92, 107, 107, 105, 84, 105, 108, 98, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828363386Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67202024-02-03T17:25:25.673Z{"msg":"[2]R generation[0..12]: [58, 61, 61, 52, 60, 60, 59, 46, 59, 61, 56, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828436843Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67212024-02-03T17:25:25.673Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828511788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67222024-02-03T17:25:25.673Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828591226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67232024-02-03T17:25:25.673Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828663676Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67242024-02-03T17:25:25.673Z{"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828738784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67252024-02-03T17:25:25.673Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828810001Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"":"downstairs","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67262024-02-03T17:25:25.673Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828885089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67272024-02-03T17:25:25.673Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.828953505Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"0","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67282024-02-03T17:25:25.673Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.829021026Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"1","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67292024-02-03T17:25:25.673Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.829103347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"client":"2","":"downstairs","session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67302024-02-03T17:25:25.673Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.829137285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67312024-02-03T17:25:25.673ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67322024-02-03T17:25:25.673Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.829183852Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67332024-02-03T17:25:25.674Z{"msg":"8b38c238-a708-447c-8861-4f45ef917086 is now active with session: b9c132fe-e292-4357-954c-4b9fae4570b3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.829208247Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67342024-02-03T17:25:25.674Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.829246874Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380,"session_id":"b9c132fe-e292-4357-954c-4b9fae4570b3"}
67352024-02-03T17:25:25.674Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:24.829269533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380}
67362024-02-03T17:25:25.674ZLoading write count information from file "/tmp/test_repair_verify.data"
67372024-02-03T17:25:25.674ZRun Repair workload
67382024-02-03T17:25:25.674Z01/30 Write block 16 len 28672 data: 4 3 3 2 3 5 5
67392024-02-03T17:25:25.678Z02/30 Write block 226 len 8192 data: 7 10
67402024-02-03T17:25:25.678Z03/30 Read block 410 len 32768
67412024-02-03T17:25:25.678Z04/30 Write block 159 len 16384 data: 4 4 5 6
67422024-02-03T17:25:25.678Z05/30 Write block 418 len 36864 data: 6 6 6 4 4 4 4 4 3
67432024-02-03T17:25:25.678Z06/30 Flush
67442024-02-03T17:25:25.682Z07/30 Write block 493 len 8192 data: 6 6
67452024-02-03T17:25:25.682Z08/30 Write block 47 len 28672 data: 4 4 6 7 7 7 6
67462024-02-03T17:25:25.682Z09/30 Read block 142 len 12288
67472024-02-03T17:25:25.687Z10/30 Write block 403 len 16384 data: 6 5 6 6
67482024-02-03T17:25:25.687Z11/30 Write block 174 len 24576 data: 4 4 4 6 5 4
67492024-02-03T17:25:25.687Z12/30 Read block 455 len 4096
67502024-02-03T17:25:25.691Z13/30 Read block 554 len 8192
67512024-02-03T17:25:25.691Z14/30 Read block 579 len 32768
67522024-02-03T17:25:25.691Z15/30 Read block 343 len 8192
67532024-02-03T17:25:25.691Z16/30 Read block 118 len 36864
67542024-02-03T17:25:25.695Z17/30 Read block 218 len 36864
67552024-02-03T17:25:25.695Z18/30 Read block 590 len 8192
67562024-02-03T17:25:25.695Z19/30 Read block 4 len 20480
67572024-02-03T17:25:25.698Z20/30 Read block 536 len 16384
67582024-02-03T17:25:25.698Z21/30 Read block 404 len 20480
67592024-02-03T17:25:25.702Z22/30 Read block 477 len 20480
67602024-02-03T17:25:25.702Z23/30 Write block 166 len 4096 data: 4
67612024-02-03T17:25:25.702Z24/30 Read block 245 len 16384
67622024-02-03T17:25:25.717Z25/30 Write block 128 len 32768 data: 8 10 10 9 8 5 7 7
67632024-02-03T17:25:25.717Z26/30 Read block 340 len 20480
67642024-02-03T17:25:25.721Z27/30 Write block 542 len 16384 data: 4 3 4 5
67652024-02-03T17:25:25.721Z28/30 Read block 230 len 40960
67662024-02-03T17:25:25.721Z29/30 Read block 343 len 28672
67672024-02-03T17:25:25.721Z30/30 Read block 232 len 32768
67682024-02-03T17:25:25.721Z----------------------------------------------------------------
67692024-02-03T17:25:25.721Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:24
67702024-02-03T17:25:25.721ZStates: Active Active Active
67712024-02-03T17:25:25.721ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67722024-02-03T17:25:25.721Z 7 Acked 1006 Write 2 Done Done Done false
67732024-02-03T17:25:25.721Z 8 Acked 1007 Write 7 Done Done Done false
67742024-02-03T17:25:25.721Z 9 Acked 1008 Read 3 Done Done Done false
67752024-02-03T17:25:25.721Z 10 Acked 1009 Write 4 Done Done Done false
67762024-02-03T17:25:25.721Z 11 Acked 1010 Write 6 Done Done Done false
67772024-02-03T17:25:25.722Z 12 Acked 1011 Read 1 Done Done Done false
67782024-02-03T17:25:25.722Z 13 Acked 1012 Read 2 Done Done Sent false
67792024-02-03T17:25:25.722Z 14 Acked 1013 Read 8 Done Done Sent false
67802024-02-03T17:25:25.722Z 15 Acked 1014 Read 2 Done Done Sent false
67812024-02-03T17:25:25.722Z 16 Acked 1015 Read 9 Done Done Sent false
67822024-02-03T17:25:25.722Z 17 Acked 1016 Read 9 Done Done Sent false
67832024-02-03T17:25:25.722Z 18 Acked 1017 Read 2 Done Done Sent false
67842024-02-03T17:25:25.722Z 19 Acked 1018 Read 5 Done Done Sent false
67852024-02-03T17:25:25.722Z 20 Acked 1019 Read 4 Done Done Sent false
67862024-02-03T17:25:25.722Z 21 Acked 1020 Read 5 Done Done Sent false
67872024-02-03T17:25:25.722Z 22 Acked 1021 Read 5 Done Done Sent false
67882024-02-03T17:25:25.722Z 23 Acked 1022 Write 1 Done Done Sent false
67892024-02-03T17:25:25.722Z 24 Acked 1023 Read 4 Done Done Sent false
67902024-02-03T17:25:25.723Z 25 Acked 1024 Write 8 Done Done Sent false
67912024-02-03T17:25:25.723Z 26 Acked 1025 Read 5 Done Done Sent false
67922024-02-03T17:25:25.723Z 27 Acked 1026 Write 4 Done Done Sent false
67932024-02-03T17:25:25.723Z 28 Acked 1027 Read 10 Done Done Sent false
67942024-02-03T17:25:25.723Z 29 Acked 1028 Read 7 Done Done Sent false
67952024-02-03T17:25:25.723Z 30 Acked 1029 Read 8 Done Sent Sent false
67962024-02-03T17:25:25.723Z STATES DS:0 DS:1 DS:2 TOTAL
67972024-02-03T17:25:25.723Z New 0 0 0 0
67982024-02-03T17:25:25.723Z Sent 0 1 18 19
67992024-02-03T17:25:25.723Z Done 24 23 6 53
68002024-02-03T17:25:25.723Z Skipped 0 0 0 0
68012024-02-03T17:25:25.723Z Error 0 0 0 0
68022024-02-03T17:25:25.723ZLast Flush: 1005 1005 1005
68032024-02-03T17:25:25.724ZDownstairs last five completed: 1005 1004 1003 1002 1001
68042024-02-03T17:25:25.724ZUpstairs last five completed: 30 29 28 27 26
68052024-02-03T17:25:25.724Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:24.874476086Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1380}
68062024-02-03T17:25:25.724ZWrote out file "/tmp/test_repair_verify.data"
68072024-02-03T17:25:25.724Z
68082024-02-03T17:25:25.727ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68092024-02-03T17:25:25.757ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68102024-02-03T17:25:25.757Z 0 000-019 62 62 62 109 109 109 F F F
68112024-02-03T17:25:25.757Z 1 020-039 62 62 62 109 109 109 F F F
68122024-02-03T17:25:25.757Z 2 040-059 61 61 61 108 108 108 T T T
68132024-02-03T17:25:25.757Z 3 060-079 52 52 52 92 92 92 F F F
68142024-02-03T17:25:25.757Z 4 080-099 60 60 60 107 107 107 F F F
68152024-02-03T17:25:25.757Z 5 100-119 60 60 60 107 107 107 F F F
68162024-02-03T17:25:25.757Z 6 120-139 59 59 59 105 105 105 T T F <---
68172024-02-03T17:25:25.757Z 7 140-159 62 62 62 109 109 109 F F F
68182024-02-03T17:25:25.761Z 8 160-179 62 62 62 109 109 109 T T T
68192024-02-03T17:25:25.761Z 9 180-199 61 61 61 108 108 108 F F F
68202024-02-03T17:25:25.761Z 10 200-219 56 56 56 98 98 98 F F F
68212024-02-03T17:25:25.761Z 11 220-239 62 62 62 109 109 109 F F F
68222024-02-03T17:25:25.761Z 12 240-259 59 59 59 105 105 105 F F F
68232024-02-03T17:25:25.761Z 13 260-279 58 58 58 102 102 102 F F F
68242024-02-03T17:25:25.761Z 14 280-299 58 58 58 104 104 104 F F F
68252024-02-03T17:25:25.761Z 15 300-319 60 60 60 106 106 106 F F F
68262024-02-03T17:25:25.761Z 16 320-339 58 58 58 100 100 100 F F F
68272024-02-03T17:25:25.762Z 17 340-359 57 57 57 99 99 99 F F F
68282024-02-03T17:25:25.762Z 18 360-379 61 61 61 108 108 108 F F F
68292024-02-03T17:25:25.762Z 19 380-399 61 61 61 108 108 108 F F F
68302024-02-03T17:25:25.762Z 20 400-419 62 62 62 109 109 109 T T T
68312024-02-03T17:25:25.762Z 21 420-439 62 62 62 109 109 109 F F F
68322024-02-03T17:25:25.762Z 22 440-459 61 61 61 108 108 108 F F F
68332024-02-03T17:25:25.762Z 23 460-479 50 50 50 89 89 89 F F F
68342024-02-03T17:25:25.762Z 24 480-499 58 58 58 101 101 101 T T T
68352024-02-03T17:25:25.762Z 25 500-519 61 61 61 108 108 108 F F F
68362024-02-03T17:25:25.762Z 26 520-539 51 51 51 90 90 90 F F F
68372024-02-03T17:25:25.762Z 27 540-559 60 60 60 106 106 106 T T F <---
68382024-02-03T17:25:25.762Z 28 560-579 55 55 55 96 96 96 F F F
68392024-02-03T17:25:25.762Z 29 580-599 55 55 55 96 96 96 F F F
68402024-02-03T17:25:25.762ZMax gen: 62, Max flush: 109
68412024-02-03T17:25:25.763ZError: Difference in extent metadata found!
68422024-02-03T17:25:25.768ZA Difference in extent metadata is expected here
68432024-02-03T17:25:25.768ZOn loop 31
68442024-02-03T17:25:25.768Z
68452024-02-03T17:25:25.768ZVerifying data now
68462024-02-03T17:25:31.399ZNo change after verify
68472024-02-03T17:25:31.399ZLoop: 31 Downstairs dump after verify (and repair):
68482024-02-03T17:25:31.424ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68492024-02-03T17:25:31.424Z 0 000-019 62 62 62 109 109 109 F F F
68502024-02-03T17:25:31.424Z 1 020-039 62 62 62 109 109 109 F F F
68512024-02-03T17:25:31.424Z 2 040-059 63 63 63 110 110 110 F F F
68522024-02-03T17:25:31.424Z 3 060-079 52 52 52 92 92 92 F F F
68532024-02-03T17:25:31.451Z 4 080-099 60 60 60 107 107 107 F F F
68542024-02-03T17:25:31.451Z 5 100-119 60 60 60 107 107 107 F F F
68552024-02-03T17:25:31.451Z 6 120-139 63 63 63 110 110 110 F F F
68562024-02-03T17:25:31.451Z 7 140-159 62 62 62 109 109 109 F F F
68572024-02-03T17:25:31.451Z 8 160-179 63 63 63 110 110 110 F F F
68582024-02-03T17:25:31.451Z 9 180-199 61 61 61 108 108 108 F F F
68592024-02-03T17:25:31.451Z 10 200-219 56 56 56 98 98 98 F F F
68602024-02-03T17:25:31.451Z 11 220-239 62 62 62 109 109 109 F F F
68612024-02-03T17:25:31.451Z 12 240-259 59 59 59 105 105 105 F F F
68622024-02-03T17:25:31.452Z 13 260-279 58 58 58 102 102 102 F F F
68632024-02-03T17:25:31.452Z 14 280-299 58 58 58 104 104 104 F F F
68642024-02-03T17:25:31.452Z 15 300-319 60 60 60 106 106 106 F F F
68652024-02-03T17:25:31.452Z 16 320-339 58 58 58 100 100 100 F F F
68662024-02-03T17:25:31.452Z 17 340-359 57 57 57 99 99 99 F F F
68672024-02-03T17:25:31.452Z 18 360-379 61 61 61 108 108 108 F F F
68682024-02-03T17:25:31.452Z 19 380-399 61 61 61 108 108 108 F F F
68692024-02-03T17:25:31.452Z 20 400-419 63 63 63 110 110 110 F F F
68702024-02-03T17:25:31.452Z 21 420-439 62 62 62 109 109 109 F F F
68712024-02-03T17:25:31.452Z 22 440-459 61 61 61 108 108 108 F F F
68722024-02-03T17:25:31.452Z 23 460-479 50 50 50 89 89 89 F F F
68732024-02-03T17:25:31.452Z 24 480-499 63 63 63 110 110 110 F F F
68742024-02-03T17:25:31.452Z 25 500-519 61 61 61 108 108 108 F F F
68752024-02-03T17:25:31.452Z 26 520-539 51 51 51 90 90 90 F F F
68762024-02-03T17:25:31.452Z 27 540-559 63 63 63 110 110 110 F F F
68772024-02-03T17:25:31.452Z 28 560-579 55 55 55 96 96 96 F F F
68782024-02-03T17:25:31.452Z 29 580-599 55 55 55 96 96 96 F F F
68792024-02-03T17:25:31.452ZMax gen: 63, Max flush: 110
68802024-02-03T17:25:31.452Z
68812024-02-03T17:25:31.452ZBegin loop 32
68822024-02-03T17:25:31.452ZDownstairs to restart: 0
68832024-02-03T17:25:31.461Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.623869046Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388}
68842024-02-03T17:25:31.462Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624322194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388}
68852024-02-03T17:25:31.462Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624391809Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388}
68862024-02-03T17:25:31.466Z{"msg":"Crucible 59fb674b-1f0f-48ee-8c0a-535ab2fdc700 has session id: 3e9f5f7d-e645-49bf-b72d-d2952624f3e6","v":0,"name":"crucible","level":30Crucible runtime is spawned
68872024-02-03T17:25:31.466Z,"time":"2024-02-03T17:25:30.624434646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68882024-02-03T17:25:31.466Z{"msg":"Upstairs opts: Upstairs UUID: 59fb674b-1f0f-48ee-8c0a-535ab2fdc700, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624541716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68892024-02-03T17:25:31.466Z{"msg":"Crucible stats registered with UUID: 59fb674b-1f0f-48ee-8c0a-535ab2fdc700","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624622919Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68902024-02-03T17:25:31.466Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624658722Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388}
68912024-02-03T17:25:31.466Z{"msg":"59fb674b-1f0f-48ee-8c0a-535ab2fdc700 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624770141Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68922024-02-03T17:25:31.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624849925Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68932024-02-03T17:25:31.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624894911Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68942024-02-03T17:25:31.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624934322Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68952024-02-03T17:25:31.467Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.624964607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68962024-02-03T17:25:31.467Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.625001227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68972024-02-03T17:25:31.467Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.625043368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68982024-02-03T17:25:31.467Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.625172892Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
68992024-02-03T17:25:31.467Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.625261865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69002024-02-03T17:25:31.467Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.625346128Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69012024-02-03T17:25:31.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.625456213Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"io task","client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69022024-02-03T17:25:31.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.625509864Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"io task","client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69032024-02-03T17:25:31.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.625552598Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"io task","client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69042024-02-03T17:25:31.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.626569067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69052024-02-03T17:25:31.467Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.626646708Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69062024-02-03T17:25:31.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.626753364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69072024-02-03T17:25:31.467Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.626838958Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69082024-02-03T17:25:31.468Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.627653242Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69092024-02-03T17:25:31.468Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.627901558Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69102024-02-03T17:25:31.472Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.628469926Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69112024-02-03T17:25:31.473Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.628543824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69122024-02-03T17:25:31.473Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.628589656Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69132024-02-03T17:25:31.473Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.628676665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69142024-02-03T17:25:31.473Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.628722212Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69152024-02-03T17:25:31.473Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.62886878Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69162024-02-03T17:25:31.473Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.628941118Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69172024-02-03T17:25:31.473Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.628969094Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69182024-02-03T17:25:31.473Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.629510481Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69192024-02-03T17:25:31.473Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630243916Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69202024-02-03T17:25:31.473Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630308668Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69212024-02-03T17:25:31.473Z{"msg":"[0]R flush_numbers[0..12]: [109, 109, 110, 92, 107, 107, 110, 109, 110, 108, 98, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630337397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69222024-02-03T17:25:31.473Z{"msg":"[0]R generation[0..12]: [62, 62, 63, 52, 60, 60, 63, 62, 63, 61, 56, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630380821Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69232024-02-03T17:25:31.473Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630407646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69242024-02-03T17:25:31.474Z{"msg":"[1]R flush_numbers[0..12]: [109, 109, 110, 92, 107, 107, 110, 109, 110, 108, 98, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630431085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69252024-02-03T17:25:31.474Z{"msg":"[1]R generation[0..12]: [62, 62, 63, 52, 60, 60, 63, 62, 63, 61, 56, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630473159Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69262024-02-03T17:25:31.474Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630498843Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69272024-02-03T17:25:31.474Z{"msg":"[2]R flush_numbers[0..12]: [109, 109, 110, 92, 107, 107, 110, 109, 110, 108, 98, 109]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
69282024-02-03T17:25:31.474Z2024-02-03T17:25:30.630541775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69292024-02-03T17:25:31.474Z{"msg":"[2]R generation[0..12]: [62, 62, 63, 52, 60, 60, 63, 62, 63, 61, 56, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630571079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69302024-02-03T17:25:31.474Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630619472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69312024-02-03T17:25:31.474Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630643605Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69322024-02-03T17:25:31.474Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630684845Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69332024-02-03T17:25:31.474Z{"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630707915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69342024-02-03T17:25:31.474Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630752282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"":"downstairs","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69352024-02-03T17:25:31.474Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630776523Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69362024-02-03T17:25:31.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630830383Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"0","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69372024-02-03T17:25:31.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630854498Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"1","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69382024-02-03T17:25:31.475Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630876236Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"client":"2","":"downstairs","session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69392024-02-03T17:25:31.475Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630918179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69402024-02-03T17:25:31.475Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630941082Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69412024-02-03T17:25:31.475Z{"msg":"59fb674b-1f0f-48ee-8c0a-535ab2fdc700 is now active with session: 3e9f5f7d-e645-49bf-b72d-d2952624f3e6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.63096272Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69422024-02-03T17:25:31.475Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.630983736Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388,"session_id":"3e9f5f7d-e645-49bf-b72d-d2952624f3e6"}
69432024-02-03T17:25:31.475Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:30.631004346Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388}
69442024-02-03T17:25:31.475ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69452024-02-03T17:25:31.475ZLoading write count information from file "/tmp/test_repair_verify.data"
69462024-02-03T17:25:31.475ZRun Repair workload
69472024-02-03T17:25:31.475Z01/30 Write block 220 len 40960 data: 6 10 10 7 7 9 8 11 9 5
69482024-02-03T17:25:31.480Z02/30 Read block 432 len 4096
69492024-02-03T17:25:31.480Z03/30 Write block 319 len 4096 data: 5
69502024-02-03T17:25:31.480Z04/30 Write block 108 len 36864 data: 7 7 7 8 8 9 10 8 6
69512024-02-03T17:25:31.480Z05/30 Read block 177 len 12288
69522024-02-03T17:25:31.480Z06/30 Flush
69532024-02-03T17:25:31.483Z07/30 Write block 493 len 32768 data: 7 7 5 5 6 6 6 6
69542024-02-03T17:25:31.483Z08/30 Flush
69552024-02-03T17:25:31.487Z09/30 Write block 272 len 8192 data: 10 10
69562024-02-03T17:25:31.492Z10/30 Write block 478 len 36864 data: 6 6 5 6 6 5 4 5 7
69572024-02-03T17:25:31.492Z11/30 Read block 136 len 16384
69582024-02-03T17:25:31.492Z12/30 Write block 336 len 4096 data: 4
69592024-02-03T17:25:31.496Z13/30 Write block 300 len 28672 data: 4 5 5 5 4 4 5
69602024-02-03T17:25:31.496Z14/30 Read block 141 len 36864
69612024-02-03T17:25:31.496Z15/30 Write block 555 len 4096 data: 6
69622024-02-03T17:25:31.496Z16/30 Read block 330 len 36864
69632024-02-03T17:25:31.500Z17/30 Write block 392 len 36864 data: 4 5 6 8 8 6 6 6 6
69642024-02-03T17:25:31.500Z18/30 Read block 589 len 24576
69652024-02-03T17:25:31.509Z19/30 Write block 204 len 24576 data: 4 5 5 7 5 4
69662024-02-03T17:25:31.509Z20/30 Write block 235 len 4096 data: 6
69672024-02-03T17:25:31.509Z21/30 Flush
69682024-02-03T17:25:31.525Z22/30 Write block 599 len 4096 data: 4
69692024-02-03T17:25:31.525Z23/30 Write block 212 len 16384 data: 4 4 6 7
69702024-02-03T17:25:31.525Z24/30 Write block 355 len 40960 data: 5 7 7 7 8 8 9 9 10 9
69712024-02-03T17:25:31.525Z25/30 Read block 126 len 4096
69722024-02-03T17:25:31.525Z26/30 Read block 40 len 16384
69732024-02-03T17:25:31.525Z27/30 Write block 269 len 40960 data: 7 10 11 11 11 10 11 13 10 9
69742024-02-03T17:25:31.549Z28/30 Read block 105 len 40960
69752024-02-03T17:25:31.549Z29/30 Read block 559 len 20480
69762024-02-03T17:25:31.549Z30/30 Write block 563 len 4096 data: 4
69772024-02-03T17:25:31.549Z----------------------------------------------------------------
69782024-02-03T17:25:31.549Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69792024-02-03T17:25:31.549ZStates: Active Active Active
69802024-02-03T17:25:31.549ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69812024-02-03T17:25:31.549Z 1 Acked 1000 Write 10 Sent Done Done false
69822024-02-03T17:25:31.549Z 2 Acked 1001 Read 1 Sent Done Done false
69832024-02-03T17:25:31.549Z 3 Acked 1002 Write 1 Sent Done Done false
69842024-02-03T17:25:31.550Z 4 Acked 1003 Write 9 Sent Done Done false
69852024-02-03T17:25:31.550Z 5 Acked 1004 Read 3 Sent Done Done false
69862024-02-03T17:25:31.550Z 6 Acked 1005 Flush 0 Sent Done Done false
69872024-02-03T17:25:31.550Z 7 Acked 1006 Write 8 Sent Done Done false
69882024-02-03T17:25:31.550Z 8 Acked 1007 Flush 0 Sent Done Done false
69892024-02-03T17:25:31.550Z 9 Acked 1008 Write 2 Sent Done Done false
69902024-02-03T17:25:31.550Z 10 Acked 1009 Write 9 Sent Done Done false
69912024-02-03T17:25:31.550Z 11 Acked 1010 Read 4 Sent Done Done false
69922024-02-03T17:25:31.550Z 12 Acked 1011 Write 1 Sent Done Done false
69932024-02-03T17:25:31.550Z 13 Acked 1012 Write 7 Sent Done Done false
69942024-02-03T17:25:31.550Z 14 Acked 1013 Read 9 Sent Done Done false
69952024-02-03T17:25:31.550Z 15 Acked 1014 Write 1 Sent Done Done false
69962024-02-03T17:25:31.550Z 16 Acked 1015 Read 9 Sent Done Done false
69972024-02-03T17:25:31.551Z 17 Acked 1016 Write 9 Sent Done Done false
69982024-02-03T17:25:31.551Z 18 Acked 1017 Read 6 Sent Done Done false
69992024-02-03T17:25:31.551Z 19 Acked 1018 Write 6 Sent Done Done false
70002024-02-03T17:25:31.551Z 20 Acked 1019 Write 1 Sent Done Done false
70012024-02-03T17:25:31.551Z 21 Acked 1020 Flush 0 Sent Done Done false
70022024-02-03T17:25:31.551Z 22 Acked 1021 Write 1 Sent Done Done false
70032024-02-03T17:25:31.551Z 23 Acked 1022 Write 4 Sent Done Done false
70042024-02-03T17:25:31.551Z 24 Acked 1023 Write 10 Sent Done Done false
70052024-02-03T17:25:31.551Z 25 Acked 1024 Read 1 Sent Done Done false
70062024-02-03T17:25:31.551Z 26 Acked 1025 Read 4 Sent Done Done false
70072024-02-03T17:25:31.551Z 27 Acked 1026 Write 10 Sent Done Done false
70082024-02-03T17:25:31.551Z 28 Acked 1027 Read 10 Sent Done Done false
70092024-02-03T17:25:31.551Z 29 Acked 1028 Read 5 Sent Sent Done false
70102024-02-03T17:25:31.552Z 30 Acked 1029 Write 1 Sent Sent Sent false
70112024-02-03T17:25:31.552Z STATES DS:0 DS:1 DS:2 TOTAL
70122024-02-03T17:25:31.552Z New 0 0 0 0
70132024-02-03T17:25:31.552Z Sent 30 2 1 33
70142024-02-03T17:25:31.552Z Done 0 28 29 57
70152024-02-03T17:25:31.552Z Skipped 0 0 0 0
70162024-02-03T17:25:31.552Z Error 0 0 0 0
70172024-02-03T17:25:31.552ZLast Flush: 0 1020 1020
70182024-02-03T17:25:31.552ZDownstairs last five completed:
70192024-02-03T17:25:31.552ZUpstairs last five completed: 30 29 28 27 26
70202024-02-03T17:25:31.552Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:30.68256079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1388}
70212024-02-03T17:25:31.552ZWrote out file "/tmp/test_repair_verify.data"
70222024-02-03T17:25:31.553Z
70232024-02-03T17:25:31.560ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70242024-02-03T17:25:31.564ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70252024-02-03T17:25:31.564Z 0 000-019 62 62 62 109 109 109 F F F
70262024-02-03T17:25:31.564Z 1 020-039 62 62 62 109 109 109 F F F
70272024-02-03T17:25:31.564Z 2 040-059 63 63 63 110 110 110 F F F
70282024-02-03T17:25:31.564Z 3 060-079 52 52 52 92 92 92 F F F
70292024-02-03T17:25:31.564Z 4 080-099 60 60 60 107 107 107 F F F
70302024-02-03T17:25:31.564Z 5 100-119 60 64 64 107 111 111 F F F <---
70312024-02-03T17:25:31.565Z 6 120-139 63 63 63 110 110 110 F F F
70322024-02-03T17:25:31.565Z 7 140-159 62 62 62 109 109 109 F F F
70332024-02-03T17:25:31.565Z 8 160-179 63 63 63 110 110 110 F F F
70342024-02-03T17:25:31.565Z 9 180-199 61 61 61 108 108 108 F F F
70352024-02-03T17:25:31.565Z 10 200-219 56 64 64 98 113 113 F T T <---
70362024-02-03T17:25:31.565Z 11 220-239 62 64 64 109 113 113 F F F <---
70372024-02-03T17:25:31.565Z 12 240-259 59 59 59 105 105 105 F F F
70382024-02-03T17:25:31.565Z 13 260-279 58 64 64 102 113 113 F T T <---
70392024-02-03T17:25:31.565Z 14 280-299 58 58 58 104 104 104 F F F
70402024-02-03T17:25:31.565Z 15 300-319 60 64 64 106 113 113 F F F <---
70412024-02-03T17:25:31.565Z 16 320-339 58 64 64 100 113 113 F F F <---
70422024-02-03T17:25:31.565Z 17 340-359 57 57 57 99 99 99 F T T <---
70432024-02-03T17:25:31.565Z 18 360-379 61 61 61 108 108 108 F T T <---
70442024-02-03T17:25:31.566Z 19 380-399 61 64 64 108 113 113 F F F <---
70452024-02-03T17:25:31.566Z 20 400-419 63 64 64 110 113 113 F F F <---
70462024-02-03T17:25:31.566Z 21 420-439 62 62 62 109 109 109 F F F
70472024-02-03T17:25:31.566Z 22 440-459 61 61 61 108 108 108 F F F
70482024-02-03T17:25:31.566Z 23 460-479 50 64 64 89 113 113 F F F <---
70492024-02-03T17:25:31.566Z 24 480-499 63 64 64 110 113 113 F F F <---
70502024-02-03T17:25:31.566Z 25 500-519 61 64 64 108 112 112 F F F <---
70512024-02-03T17:25:31.566Z 26 520-539 51 51 51 90 90 90 F F F
70522024-02-03T17:25:31.566Z 27 540-559 63 64 64 110 113 113 F F F <---
70532024-02-03T17:25:31.566Z 28 560-579 55 55 55 96 96 96 F T T <---
70542024-02-03T17:25:31.566Z 29 580-599 55 55 55 96 96 96 F T T <---
70552024-02-03T17:25:31.566ZMax gen: 64, Max flush: 113
70562024-02-03T17:25:31.566ZError: Difference in extent metadata found!
70572024-02-03T17:25:31.570ZA Difference in extent metadata is expected here
70582024-02-03T17:25:31.570ZOn loop 32
70592024-02-03T17:25:31.570Z
70602024-02-03T17:25:31.573ZVerifying data now
70612024-02-03T17:25:36.961ZNo change after verify
70622024-02-03T17:25:36.961ZLoop: 32 Downstairs dump after verify (and repair):
70632024-02-03T17:25:36.986ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70642024-02-03T17:25:36.986Z 0 000-019 62 62 62 109 109 109 F F F
70652024-02-03T17:25:36.986Z 1 020-039 62 62 62 109 109 109 F F F
70662024-02-03T17:25:36.986Z 2 040-059 63 63 63 110 110 110 F F F
70672024-02-03T17:25:36.986Z 3 060-079 52 52 52 92 92 92 F F F
70682024-02-03T17:25:36.986Z 4 080-099 60 60 60 107 107 107 F F F
70692024-02-03T17:25:36.986Z 5 100-119 64 64 64 111 111 111 F F F
70702024-02-03T17:25:36.986Z 6 120-139 63 63 63 110 110 110 F F F
70712024-02-03T17:25:36.986Z 7 140-159 62 62 62 109 109 109 F F F
70722024-02-03T17:25:36.989Z 8 160-179 63 63 63 110 110 110 F F F
70732024-02-03T17:25:36.989Z 9 180-199 61 61 61 108 108 108 F F F
70742024-02-03T17:25:36.990Z 10 200-219 65 65 65 114 114 114 F F F
70752024-02-03T17:25:36.990Z 11 220-239 64 64 64 113 113 113 F F F
70762024-02-03T17:25:36.990Z 12 240-259 59 59 59 105 105 105 F F F
70772024-02-03T17:25:36.990Z 13 260-279 65 65 65 114 114 114 F F F
70782024-02-03T17:25:36.990Z 14 280-299 58 58 58 104 104 104 F F F
70792024-02-03T17:25:36.990Z 15 300-319 64 64 64 113 113 113 F F F
70802024-02-03T17:25:36.990Z 16 320-339 64 64 64 113 113 113 F F F
70812024-02-03T17:25:36.990Z 17 340-359 65 65 65 114 114 114 F F F
70822024-02-03T17:25:36.990Z 18 360-379 65 65 65 114 114 114 F F F
70832024-02-03T17:25:36.990Z 19 380-399 64 64 64 113 113 113 F F F
70842024-02-03T17:25:36.990Z 20 400-419 64 64 64 113 113 113 F F F
70852024-02-03T17:25:36.990Z 21 420-439 62 62 62 109 109 109 F F F
70862024-02-03T17:25:36.990Z 22 440-459 61 61 61 108 108 108 F F F
70872024-02-03T17:25:36.990Z 23 460-479 64 64 64 113 113 113 F F F
70882024-02-03T17:25:36.990Z 24 480-499 64 64 64 113 113 113 F F F
70892024-02-03T17:25:36.990Z 25 500-519 64 64 64 112 112 112 F F F
70902024-02-03T17:25:36.990Z 26 520-539 51 51 51 90 90 90 F F F
70912024-02-03T17:25:36.990Z 27 540-559 64 64 64 113 113 113 F F F
70922024-02-03T17:25:36.990Z 28 560-579 65 65 65 114 114 114 F F F
70932024-02-03T17:25:36.990Z 29 580-599 65 65 65 114 114 114 F F F
70942024-02-03T17:25:36.990ZMax gen: 65, Max flush: 114
70952024-02-03T17:25:36.994Z
70962024-02-03T17:25:36.994ZBegin loop 33
70972024-02-03T17:25:36.994ZDownstairs to restart: 0
70982024-02-03T17:25:37.038Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.186761725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396}
70992024-02-03T17:25:37.038Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.187300808Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396}
71002024-02-03T17:25:37.038Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.187329901Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396}
71012024-02-03T17:25:37.047Z{"msg":"Crucible eb0fd02c-fbd9-4c0c-87ea-528e6783a088 has session id: a2cf8668-faf0-4ff5-97cc-e2e270a4dac4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.187364423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71022024-02-03T17:25:37.047Z{"msg":"Upstairs opts: Upstairs UUID: eb0fd02c-fbd9-4c0c-87ea-528e6783a088, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.187534691Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71032024-02-03T17:25:37.047Z{"msg":"Crucible stats registered with UUID: eb0fd02c-fbd9-4c0c-87ea-528e6783a088","v":0,"name":"crucible","level":30Crucible runtime is spawned
71042024-02-03T17:25:37.047Z,"time":"2024-02-03T17:25:36.187641185Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71052024-02-03T17:25:37.048Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.187780625Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396}
71062024-02-03T17:25:37.048Z{"msg":"eb0fd02c-fbd9-4c0c-87ea-528e6783a088 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188084936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71072024-02-03T17:25:37.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.18815667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71082024-02-03T17:25:37.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188184432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71092024-02-03T17:25:37.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188229294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71102024-02-03T17:25:37.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188254212Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71112024-02-03T17:25:37.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188276285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71122024-02-03T17:25:37.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188299922Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71132024-02-03T17:25:37.048Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188321755Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71142024-02-03T17:25:37.048Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188344983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71152024-02-03T17:25:37.048Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188381244Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71162024-02-03T17:25:37.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188665799Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"io task","client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71172024-02-03T17:25:37.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188832472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"io task","client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71182024-02-03T17:25:37.049Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.188905422Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"io task","client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71192024-02-03T17:25:37.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.190130695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71202024-02-03T17:25:37.049Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.190206889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71212024-02-03T17:25:37.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.190261924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71222024-02-03T17:25:37.049Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.190305445Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71232024-02-03T17:25:37.049Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.191288768Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71242024-02-03T17:25:37.049Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.191573242Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71252024-02-03T17:25:37.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.191652285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71262024-02-03T17:25:37.049Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.19169671Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71272024-02-03T17:25:37.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.192137013Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71282024-02-03T17:25:37.049Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.192254115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71292024-02-03T17:25:37.049Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.192305987Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71302024-02-03T17:25:37.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.192494921Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71312024-02-03T17:25:37.049Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.192581473Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71322024-02-03T17:25:37.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.192663405Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71332024-02-03T17:25:37.050Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.192736701Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71342024-02-03T17:25:37.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.193336557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71352024-02-03T17:25:37.050Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.193418027Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71362024-02-03T17:25:37.050Z{"msg":"[0]R flush_numbers[0..12]: [109, 109, 110, 92, 107, 111, 110, 109, 110, 108, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.193464485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71372024-02-03T17:25:37.050Z{"msg":"[0]R generation[0..12]: [62, 62, 63, 52, 60, 64, 63, 62, 63, 61, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.19353499Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71382024-02-03T17:25:37.050Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.193578927Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71392024-02-03T17:25:37.050Z{"msg":"[1]R flush_numbers[0..12]: [109, 109, 110, 92, 107, 111, 110, 109, 110, 108, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.193646867Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71402024-02-03T17:25:37.050Z{"msg":"[1]R generation[0..12]: [62, 62, 63, 52, 60, 64, 63, 62, 63, 61, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.193722572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO
71412024-02-03T17:25:37.050Z:"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71422024-02-03T17:25:37.050Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.193863939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71432024-02-03T17:25:37.050Z{"msg":"[2]R flush_numbers[0..12]: [109, 109, 110, 92, 107, 111, 110, 109, 110, 108, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.193943715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71442024-02-03T17:25:37.050Z{"msg":"[2]R generation[0..12]: [62, 62, 63, 52, 60, 64, 63, 62, 63, 61, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194020177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71452024-02-03T17:25:37.050Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.19410789Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71462024-02-03T17:25:37.051Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194180092Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71472024-02-03T17:25:37.051Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194251944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71482024-02-03T17:25:37.051Z{"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194322508Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71492024-02-03T17:25:37.051Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194393397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"":"downstairs","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71502024-02-03T17:25:37.051Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194464667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71512024-02-03T17:25:37.051Z"}
71522024-02-03T17:25:37.051Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194564029Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"0","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71532024-02-03T17:25:37.051Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194597097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"1","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71542024-02-03T17:25:37.051Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194641737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"client":"2","":"downstairs","session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71552024-02-03T17:25:37.051Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194666372Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71562024-02-03T17:25:37.051Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194686988Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71572024-02-03T17:25:37.051Z{"msg":"eb0fd02c-fbd9-4c0c-87ea-528e6783a088 is now active with session: a2cf8668-faf0-4ff5-97cc-e2e270a4dac4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194727326Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71582024-02-03T17:25:37.051Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194749112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396,"session_id":"a2cf8668-faf0-4ff5-97cc-e2e270a4dac4"}
71592024-02-03T17:25:37.051Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:36.194788591Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396}
71602024-02-03T17:25:37.052ZLoading write count information from file "/tmp/test_repair_verify.data"
71612024-02-03T17:25:37.052ZRun Repair workload
71622024-02-03T17:25:37.052Z01/30 Write block 500 len 24576 data: 7 8 8 8 8 11
71632024-02-03T17:25:37.052Z02/30 Read block 144 len 32768
71642024-02-03T17:25:37.052Z03/30 Read block 136 len 12288
71652024-02-03T17:25:37.052Z04/30 Read block 457 len 4096
71662024-02-03T17:25:37.052Z05/30 Read block 456 len 20480
71672024-02-03T17:25:37.052Z06/30 Flush
71682024-02-03T17:25:37.057Z07/30 Write block 344 len 20480 data: 5 5 5 6 5
71692024-02-03T17:25:37.057Z08/30 Read block 539 len 8192
71702024-02-03T17:25:37.057Z09/30 Write block 356 len 40960 data: 8 8 8 9 9 10 10 11 10 8
71712024-02-03T17:25:37.057Z10/30 Read block 202 len 36864
71722024-02-03T17:25:37.057Z11/30 Write block 403 len 12288 data: 7 6 7
71732024-02-03T17:25:37.057Z12/30 Flush
71742024-02-03T17:25:37.057Z13/30 Write block 62 len 24576 data: 6 6 6 5 5 4
71752024-02-03T17:25:37.066Z14/30 Read block 277 len 4096
71762024-02-03T17:25:37.066Z15/30 Flush
71772024-02-03T17:25:37.066Z16/30 Write block 268 len 28672 data: 6 8 11 12 12 12 11
71782024-02-03T17:25:37.071Z17/30 Read block 367 len 24576
71792024-02-03T17:25:37.071Z18/30 Write block 529 len 20480 data: 8 7 6 6 5
71802024-02-03T17:25:37.071Z19/30 Write block 399 len 36864 data: 7 7 5 6 8 7 8 7 6
71812024-02-03T17:25:37.071Z20/30 Write block 313 len 4096 data: 3
71822024-02-03T17:25:37.071Z21/30 Write block 487 len 28672 data: 8 7 7 6 6 6 8
71832024-02-03T17:25:37.071Z22/30 Write block 266 len 4096 data: 6
71842024-02-03T17:25:37.071Z23/30 Write block 126 len 24576 data: 9 8 9 11 11 10
71852024-02-03T17:25:37.071Z24/30 Flush
71862024-02-03T17:25:37.077Z25/30 Write block 279 len 24576 data: 6 4 6 6 6 7
71872024-02-03T17:25:37.077Z26/30 Flush
71882024-02-03T17:25:37.092Z27/30 Write block 167 len 24576 data: 5 4 7 7 6 5
71892024-02-03T17:25:37.092Z28/30 Write block 158 len 28672 data: 4 5 5 6 7 5 6
71902024-02-03T17:25:37.092Z29/30 Write block 292 len 36864 data: 7 8 7 5 5 6 5 3 5
71912024-02-03T17:25:37.092Z30/30 Read block 355 len 8192
71922024-02-03T17:25:37.092Z----------------------------------------------------------------
71932024-02-03T17:25:37.092Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
71942024-02-03T17:25:37.092ZStates: Active Active Active
71952024-02-03T17:25:37.092ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71962024-02-03T17:25:37.092Z 1 Acked 1000 Write 6 Sent Done Done false
71972024-02-03T17:25:37.092Z 2 Acked 1001 Read 8 Sent Done Done false
71982024-02-03T17:25:37.092Z 3 Acked 1002 Read 3 Sent Done Done false
71992024-02-03T17:25:37.092Z 4 Acked 1003 Read 1 Sent Done Done false
72002024-02-03T17:25:37.092Z 5 Acked 1004 Read 5 Sent Done Done false
72012024-02-03T17:25:37.093Z 6 Acked 1005 Flush 0 Sent Done Done false
72022024-02-03T17:25:37.093Z 7 Acked 1006 Write 5 Sent Done Done false
72032024-02-03T17:25:37.093Z 8 Acked 1007 Read 2 Sent Done Done false
72042024-02-03T17:25:37.093Z 9 Acked 1008 Write 10 Sent Done Done false
72052024-02-03T17:25:37.093Z 10 Acked 1009 Read 9 Sent Done Done false
72062024-02-03T17:25:37.093Z 11 Acked 1010 Write 3 Sent Done Done false
72072024-02-03T17:25:37.093Z 12 Acked 1011 Flush 0 Sent Done Done false
72082024-02-03T17:25:37.093Z 13 Acked 1012 Write 6 Sent Done Done false
72092024-02-03T17:25:37.093Z 14 Acked 1013 Read 1 Sent Done Done false
72102024-02-03T17:25:37.093Z 15 Acked 1014 Flush 0 Sent Done Done false
72112024-02-03T17:25:37.093Z 16 Acked 1015 Write 7 Sent Done Done false
72122024-02-03T17:25:37.093Z 17 Acked 1016 Read 6 Sent Done Done false
72132024-02-03T17:25:37.093Z 18 Acked 1017 Write 5 Sent Done Done false
72142024-02-03T17:25:37.093Z 19 Acked 1018 Write 9 Sent Done Done false
72152024-02-03T17:25:37.093Z 20 Acked 1019 Write 1 Sent Done Done false
72162024-02-03T17:25:37.093Z 21 Acked 1020 Write 7 Sent Done Done false
72172024-02-03T17:25:37.093Z 22 Acked 1021 Write 1 Sent Done Done false
72182024-02-03T17:25:37.093Z 23 Acked 1022 Write 6 Sent Done Done false
72192024-02-03T17:25:37.093Z 24 Acked 1023 Flush 0 Sent Done Done false
72202024-02-03T17:25:37.093Z 25 Acked 1024 Write 6 Sent Done Done false
72212024-02-03T17:25:37.093Z 26 Acked 1025 Flush 0 Sent Done Done false
72222024-02-03T17:25:37.093Z 27 Acked 1026 Write 6 Sent Done Done false
72232024-02-03T17:25:37.093Z 28 Acked 1027 Write 7 Sent Done Done false
72242024-02-03T17:25:37.093Z 29 Acked 1028 Write 9 Sent Done Done false
72252024-02-03T17:25:37.094Z 30 Acked 1029 Read 2 Sent Sent Done false
72262024-02-03T17:25:37.094Z STATES DS:0 DS:1 DS:2 TOTAL
72272024-02-03T17:25:37.094Z New 0 0 0 0
72282024-02-03T17:25:37.094Z Sent 30 1 0 31
72292024-02-03T17:25:37.094Z Done 0 29 30 59
72302024-02-03T17:25:37.094Z Skipped 0 0 0 0
72312024-02-03T17:25:37.094Z Error 0 0 0 0
72322024-02-03T17:25:37.094ZLast Flush: 0 1025 1025
72332024-02-03T17:25:37.094ZDownstairs last five completed:
72342024-02-03T17:25:37.094ZUpstairs last five completed: 30 29 28 27 26
72352024-02-03T17:25:37.094Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:36.247197289Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1396}
72362024-02-03T17:25:37.094ZWrote out file "/tmp/test_repair_verify.data"
72372024-02-03T17:25:37.100Z
72382024-02-03T17:25:37.104ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72392024-02-03T17:25:37.127ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72402024-02-03T17:25:37.128Z 0 000-019 62 62 62 109 109 109 F F F
72412024-02-03T17:25:37.128Z 1 020-039 62 62 62 109 109 109 F F F
72422024-02-03T17:25:37.128Z 2 040-059 63 63 63 110 110 110 F F F
72432024-02-03T17:25:37.133Z 3 060-079 52 66 66 92 117 117 F F F <---
72442024-02-03T17:25:37.133Z 4 080-099 60 60 60 107 107 107 F F F
72452024-02-03T17:25:37.133Z 5 100-119 64 64 64 111 111 111 F F F
72462024-02-03T17:25:37.133Z 6 120-139 63 66 66 110 118 118 F F F <---
72472024-02-03T17:25:37.134Z 7 140-159 62 62 62 109 109 109 F T T <---
72482024-02-03T17:25:37.134Z 8 160-179 63 63 63 110 110 110 F T T <---
72492024-02-03T17:25:37.134Z 9 180-199 61 61 61 108 108 108 F F F
72502024-02-03T17:25:37.134Z 10 200-219 65 65 65 114 114 114 F F F
72512024-02-03T17:25:37.134Z 11 220-239 64 64 64 113 113 113 F F F
72522024-02-03T17:25:37.134Z 12 240-259 59 59 59 105 105 105 F F F
72532024-02-03T17:25:37.134Z 13 260-279 65 66 66 114 119 119 F F F <---
72542024-02-03T17:25:37.134Z 14 280-299 58 66 66 104 119 119 F T T <---
72552024-02-03T17:25:37.134Z 15 300-319 64 66 66 113 118 118 F T T <---
72562024-02-03T17:25:37.134Z 16 320-339 64 64 64 113 113 113 F F F
72572024-02-03T17:25:37.134Z 17 340-359 65 66 66 114 116 116 F F F <---
72582024-02-03T17:25:37.134Z 18 360-379 65 66 66 114 116 116 F F F <---
72592024-02-03T17:25:37.134Z 19 380-399 64 66 66 113 118 118 F F F <---
72602024-02-03T17:25:37.134Z 20 400-419 64 66 66 113 118 118 F F F <---
72612024-02-03T17:25:37.134Z 21 420-439 62 62 62 109 109 109 F F F
72622024-02-03T17:25:37.134Z 22 440-459 61 61 61 108 108 108 F F F
72632024-02-03T17:25:37.134Z 23 460-479 64 64 64 113 113 113 F F F
72642024-02-03T17:25:37.134Z 24 480-499 64 66 66 113 118 118 F F F <---
72652024-02-03T17:25:37.134Z 25 500-519 64 66 66 112 115 115 F F F <---
72662024-02-03T17:25:37.134Z 26 520-539 51 66 66 90 118 118 F F F <---
72672024-02-03T17:25:37.134Z 27 540-559 64 64 64 113 113 113 F F F
72682024-02-03T17:25:37.134Z 28 560-579 65 65 65 114 114 114 F F F
72692024-02-03T17:25:37.134Z 29 580-599 65 65 65 114 114 114 F F F
72702024-02-03T17:25:37.134ZMax gen: 66, Max flush: 119
72712024-02-03T17:25:37.135ZError: Difference in extent metadata found!
72722024-02-03T17:25:37.141ZA Difference in extent metadata is expected here
72732024-02-03T17:25:37.141ZOn loop 33
72742024-02-03T17:25:37.141Z
72752024-02-03T17:25:37.141ZVerifying data now
72762024-02-03T17:25:42.350ZNo change after verify
72772024-02-03T17:25:42.350ZLoop: 33 Downstairs dump after verify (and repair):
72782024-02-03T17:25:42.375ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72792024-02-03T17:25:42.375Z 0 000-019 62 62 62 109 109 109 F F F
72802024-02-03T17:25:42.375Z 1 020-039 62 62 62 109 109 109 F F F
72812024-02-03T17:25:42.375Z 2 040-059 63 63 63 110 110 110 F F F
72822024-02-03T17:25:42.375Z 3 060-079 66 66 66 117 117 117 F F F
72832024-02-03T17:25:42.376Z 4 080-099 60 60 60 107 107 107 F F F
72842024-02-03T17:25:42.376Z 5 100-119 64 64 64 111 111 111 F F F
72852024-02-03T17:25:42.376Z 6 120-139 66 66 66 118 118 118 F F F
72862024-02-03T17:25:42.376Z 7 140-159 67 67 67 120 120 120 F F F
72872024-02-03T17:25:42.376Z 8 160-179 67 67 67 120 120 120 F F F
72882024-02-03T17:25:42.376Z 9 180-199 61 61 61 108 108 108 F F F
72892024-02-03T17:25:42.380Z 10 200-219 65 65 65 114 114 114 F F F
72902024-02-03T17:25:42.380Z 11 220-239 64 64 64 113 113 113 F F F
72912024-02-03T17:25:42.380Z 12 240-259 59 59 59 105 105 105 F F F
72922024-02-03T17:25:42.380Z 13 260-279 66 66 66 119 119 119 F F F
72932024-02-03T17:25:42.380Z 14 280-299 67 67 67 120 120 120 F F F
72942024-02-03T17:25:42.380Z 15 300-319 67 67 67 120 120 120 F F F
72952024-02-03T17:25:42.380Z 16 320-339 64 64 64 113 113 113 F F F
72962024-02-03T17:25:42.381Z 17 340-359 66 66 66 116 116 116 F F F
72972024-02-03T17:25:42.381Z 18 360-379 66 66 66 116 116 116 F F F
72982024-02-03T17:25:42.381Z 19 380-399 66 66 66 118 118 118 F F F
72992024-02-03T17:25:42.381Z 20 400-419 66 66 66 118 118 118 F F F
73002024-02-03T17:25:42.381Z 21 420-439 62 62 62 109 109 109 F F F
73012024-02-03T17:25:42.381Z 22 440-459 61 61 61 108 108 108 F F F
73022024-02-03T17:25:42.381Z 23 460-479 64 64 64 113 113 113 F F F
73032024-02-03T17:25:42.381Z 24 480-499 66 66 66 118 118 118 F F F
73042024-02-03T17:25:42.381Z 25 500-519 66 66 66 115 115 115 F F F
73052024-02-03T17:25:42.381Z 26 520-539 66 66 66 118 118 118 F F F
73062024-02-03T17:25:42.381Z 27 540-559 64 64 64 113 113 113 F F F
73072024-02-03T17:25:42.381Z 28 560-579 65 65 65 114 114 114 F F F
73082024-02-03T17:25:42.382Z 29 580-599 65 65 65 114 114 114 F F F
73092024-02-03T17:25:42.382ZMax gen: 67, Max flush: 120
73102024-02-03T17:25:42.382Z
73112024-02-03T17:25:42.382ZBegin loop 34
73122024-02-03T17:25:42.382ZDownstairs to restart: 2
73132024-02-03T17:25:42.416Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.578227203Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404}
73142024-02-03T17:25:42.416Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.578760398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404}
73152024-02-03T17:25:42.416Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.578807245Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404}
73162024-02-03T17:25:42.421Z{"msg":"Crucible 0d1391c9-aba7-4e73-94b0-6755b84d72b5 has session id: 7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.578878173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73172024-02-03T17:25:42.421Z{"msg":"Upstairs opts: Upstairs UUID: 0d1391c9-aba7-4e73-94b0-6755b84d72b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.578968839Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73182024-02-03T17:25:42.421Z{"msg":"Crucible stats registered with UUID: 0d1391c9-aba7-4e73-94b0-6755b84d72b5"Crucible runtime is spawned
73192024-02-03T17:25:42.421Z,"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579089133Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73202024-02-03T17:25:42.421Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579228218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404}
73212024-02-03T17:25:42.421Z{"msg":"0d1391c9-aba7-4e73-94b0-6755b84d72b5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579295906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73222024-02-03T17:25:42.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579338565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73232024-02-03T17:25:42.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579410087Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73242024-02-03T17:25:42.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579451968Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73252024-02-03T17:25:42.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579492517Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73262024-02-03T17:25:42.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579548825Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73272024-02-03T17:25:42.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579666959Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73282024-02-03T17:25:42.422Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579732403Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73292024-02-03T17:25:42.422Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579770018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73302024-02-03T17:25:42.422Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579839979Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73312024-02-03T17:25:42.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579912792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"io task","client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73322024-02-03T17:25:42.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.579957867Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"io task","client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73332024-02-03T17:25:42.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.580003753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"io task","client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73342024-02-03T17:25:42.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.581173096Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73352024-02-03T17:25:42.422Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.581230729Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73362024-02-03T17:25:42.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.581273037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73372024-02-03T17:25:42.422Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.581362915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73382024-02-03T17:25:42.422Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.58238798Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73392024-02-03T17:25:42.422Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.582617026Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73402024-02-03T17:25:42.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.58284363Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73412024-02-03T17:25:42.423Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.582905786Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73422024-02-03T17:25:42.427Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.583274042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73432024-02-03T17:25:42.427Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.583351826Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73442024-02-03T17:25:42.428Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.583396866Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73452024-02-03T17:25:42.428Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.583439025Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73462024-02-03T17:25:42.428Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.583478975Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73472024-02-03T17:25:42.428Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.583564037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73482024-02-03T17:25:42.428Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.583793091Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73492024-02-03T17:25:42.428Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584490147Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73502024-02-03T17:25:42.428Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584555208Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73512024-02-03T17:25:42.428Z{"msg":"[0]R flush_numbers[0..12]: [109, 109, 110, 117, 107, 111, 118, 120, 120, 108, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584582613Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73522024-02-03T17:25:42.428Z{"msg":"[0]R generation[0..12]: [62, 62, 63, 66, 60, 64, 66, 67, 67, 61, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.58464505Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73532024-02-03T17:25:42.428Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584706692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73542024-02-03T17:25:42.428Z{"msg":"[1]R flush_numbers[0..12]: [109, 109, 110, 117, 107, 111, 118, 120, 120, 108, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584768103Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73552024-02-03T17:25:42.428Z{"msg":"[1]R generation[0..12]: [62, 62, 63, 66, 60, 64, 66, 67, 67, 61, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584799056Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73562024-02-03T17:25:42.428ZWait for a query_work_queue command to finish before sending IO
73572024-02-03T17:25:42.429Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584862636Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73582024-02-03T17:25:42.429Z{"msg":"[2]R flush_numbers[0..12]: [109, 109, 110, 117, 107, 111, 118, 120, 120, 108, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584924567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73592024-02-03T17:25:42.429Z{"msg":"[2]R generation[0..12]: [62, 62, 63, 66, 60, 64, 66, 67, 67, 61, 65, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.584953495Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73602024-02-03T17:25:42.429Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585014501Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73612024-02-03T17:25:42.429Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585041376Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73622024-02-03T17:25:42.429Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585129124Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73632024-02-03T17:25:42.429Z{"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585197728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73642024-02-03T17:25:42.429Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585276698Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"":"downstairs","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73652024-02-03T17:25:42.429Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585314613Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73662024-02-03T17:25:42.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.58533666Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"0","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73672024-02-03T17:25:42.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585358678Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"1","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73682024-02-03T17:25:42.429Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585380123Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"client":"2","":"downstairs","session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73692024-02-03T17:25:42.429Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585401502Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73702024-02-03T17:25:42.430Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585426909Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73712024-02-03T17:25:42.430Z{"msg":"0d1391c9-aba7-4e73-94b0-6755b84d72b5 is now active with session: 7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585465833Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73722024-02-03T17:25:42.430Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.58552502Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404,"session_id":"7f0c94e5-8cd7-4a49-9e4d-d40a55e24ead"}
73732024-02-03T17:25:42.430Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:41.585549392Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404}
73742024-02-03T17:25:42.430ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73752024-02-03T17:25:42.430ZLoading write count information from file "/tmp/test_repair_verify.data"
73762024-02-03T17:25:42.430ZRun Repair workload
73772024-02-03T17:25:42.430Z01/30 Write block 321 len 16384 data: 4 6 5 5
73782024-02-03T17:25:42.430Z02/30 Read block 46 len 36864
73792024-02-03T17:25:42.434Z03/30 Flush
73802024-02-03T17:25:42.434Z04/30 Write block 203 len 36864 data: 6 5 6 6 8 6 5 6 5
73812024-02-03T17:25:42.434Z05/30 Flush
73822024-02-03T17:25:42.438Z06/30 Write block 72 len 28672 data: 2 2 3 4 4 4 4
73832024-02-03T17:25:42.438Z07/30 Read block 505 len 16384
73842024-02-03T17:25:42.441Z08/30 Read block 227 len 28672
73852024-02-03T17:25:42.442Z09/30 Flush
73862024-02-03T17:25:42.447Z10/30 Write block 553 len 28672 data: 7 7 7 7 6 6 7
73872024-02-03T17:25:42.447Z11/30 Write block 261 len 28672 data: 7 6 5 6 5 7 4
73882024-02-03T17:25:42.447Z12/30 Read block 490 len 24576
73892024-02-03T17:25:42.450Z13/30 Flush
73902024-02-03T17:25:42.454Z14/30 Write block 307 len 20480 data: 3 4 4 4 3
73912024-02-03T17:25:42.454Z15/30 Read block 47 len 36864
73922024-02-03T17:25:42.454Z16/30 Read block 590 len 4096
73932024-02-03T17:25:42.458Z17/30 Write block 444 len 28672 data: 5 6 6 6 7 7 7
73942024-02-03T17:25:42.458Z18/30 Write block 538 len 16384 data: 5 5 5 4
73952024-02-03T17:25:42.458Z19/30 Read block 87 len 32768
73962024-02-03T17:25:42.464Z20/30 Read block 136 len 28672
73972024-02-03T17:25:42.465Z21/30 Write block 349 len 8192 data: 5 5
73982024-02-03T17:25:42.465Z22/30 Read block 395 len 28672
73992024-02-03T17:25:42.465Z23/30 Write block 523 len 8192 data: 3 3
74002024-02-03T17:25:42.465Z24/30 Write block 77 len 36864 data: 5 5 3 3 3 4 6 7 5
74012024-02-03T17:25:42.465Z25/30 Read block 380 len 24576
74022024-02-03T17:25:42.486Z26/30 Write block 593 len 20480 data: 7 7 7 6 6
74032024-02-03T17:25:42.486Z27/30 Read block 211 len 24576
74042024-02-03T17:25:42.491Z28/30 Read block 271 len 12288
74052024-02-03T17:25:42.491Z29/30 Write block 197 len 8192 data: 4 4
74062024-02-03T17:25:42.491Z30/30 Write block 10 len 20480 data: 9 8 7 7 5
74072024-02-03T17:25:42.491Z----------------------------------------------------------------
74082024-02-03T17:25:42.491Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:25
74092024-02-03T17:25:42.491ZStates: Active Active Active
74102024-02-03T17:25:42.491ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74112024-02-03T17:25:42.491Z 6 Acked 1005 Write 7 Done Done Done false
74122024-02-03T17:25:42.491Z 7 Acked 1006 Read 4 Done Done Sent false
74132024-02-03T17:25:42.491Z 8 Acked 1007 Read 7 Done Done Sent false
74142024-02-03T17:25:42.491Z 9 Acked 1008 Flush 0 Done Done Sent false
74152024-02-03T17:25:42.491Z 10 Acked 1009 Write 7 Done Done Sent false
74162024-02-03T17:25:42.492Z 11 Acked 1010 Write 7 Done Done Sent false
74172024-02-03T17:25:42.492Z 12 Acked 1011 Read 6 Done Done Sent false
74182024-02-03T17:25:42.492Z 13 Acked 1012 Flush 0 Done Done Sent false
74192024-02-03T17:25:42.492Z 14 Acked 1013 Write 5 Done Done Sent false
74202024-02-03T17:25:42.492Z 15 Acked 1014 Read 9 Done Done Sent false
74212024-02-03T17:25:42.492Z 16 Acked 1015 Read 1 Done Done Sent false
74222024-02-03T17:25:42.492Z 17 Acked 1016 Write 7 Done Done Sent false
74232024-02-03T17:25:42.492Z 18 Acked 1017 Write 4 Done Done Sent false
74242024-02-03T17:25:42.492Z 19 Acked 1018 Read 8 Done Done Sent false
74252024-02-03T17:25:42.492Z 20 Acked 1019 Read 7 Done Done Sent false
74262024-02-03T17:25:42.492Z 21 Acked 1020 Write 2 Done Done Sent false
74272024-02-03T17:25:42.492Z 22 Acked 1021 Read 7 Done Done Sent false
74282024-02-03T17:25:42.492Z 23 Acked 1022 Write 2 Done Done Sent false
74292024-02-03T17:25:42.492Z 24 Acked 1023 Write 9 Done Done Sent false
74302024-02-03T17:25:42.493Z 25 Acked 1024 Read 6 Done Done Sent false
74312024-02-03T17:25:42.493Z 26 Acked 1025 Write 5 Done Done Sent false
74322024-02-03T17:25:42.493Z 27 Acked 1026 Read 6 Done Done Sent false
74332024-02-03T17:25:42.493Z 28 Acked 1027 Read 3 Done Done Sent false
74342024-02-03T17:25:42.493Z 29 Acked 1028 Write 2 Done Done Sent false
74352024-02-03T17:25:42.493Z 30 Acked 1029 Write 5 Sent Sent Sent false
74362024-02-03T17:25:42.493Z STATES DS:0 DS:1 DS:2 TOTAL
74372024-02-03T17:25:42.493Z New 0 0 0 0
74382024-02-03T17:25:42.493Z Sent 1 1 24 26
74392024-02-03T17:25:42.493Z Done 24 24 1 49
74402024-02-03T17:25:42.493Z Skipped 0 0 0 0
74412024-02-03T17:25:42.493Z Error 0 0 0 0
74422024-02-03T17:25:42.493ZLast Flush: 1012 1012 1004
74432024-02-03T17:25:42.493ZDownstairs last five completed: 1004 1003 1002 1001 1000
74442024-02-03T17:25:42.494ZUpstairs last five completed: 30 29 28 27 26
74452024-02-03T17:25:42.494Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:41.636261173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1404}
74462024-02-03T17:25:42.494ZWrote out file "/tmp/test_repair_verify.data"
74472024-02-03T17:25:42.494Z
74482024-02-03T17:25:42.494ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74492024-02-03T17:25:42.535ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74502024-02-03T17:25:42.536Z 0 000-019 62 62 62 109 109 109 T T F <---
74512024-02-03T17:25:42.536Z 1 020-039 62 62 62 109 109 109 F F F
74522024-02-03T17:25:42.536Z 2 040-059 63 63 63 110 110 110 F F F
74532024-02-03T17:25:42.549Z 3 060-079 68 68 66 123 123 117 T T T <---
74542024-02-03T17:25:42.549Z 4 080-099 60 60 60 107 107 107 T T F <---
74552024-02-03T17:25:42.549Z 5 100-119 64 64 64 111 111 111 F F F
74562024-02-03T17:25:42.549Z 6 120-139 66 66 66 118 118 118 F F F
74572024-02-03T17:25:42.549Z 7 140-159 67 67 67 120 120 120 F F F
74582024-02-03T17:25:42.549Z 8 160-179 67 67 67 120 120 120 F F F
74592024-02-03T17:25:42.549Z 9 180-199 61 61 61 108 108 108 T T F <---
74602024-02-03T17:25:42.549Z 10 200-219 68 68 68 122 122 122 F F F
74612024-02-03T17:25:42.549Z 11 220-239 64 64 64 113 113 113 F F F
74622024-02-03T17:25:42.550Z 12 240-259 59 59 59 105 105 105 F F F
74632024-02-03T17:25:42.550Z 13 260-279 68 68 66 124 124 119 F F F <---
74642024-02-03T17:25:42.550Z 14 280-299 67 67 67 120 120 120 F F F
74652024-02-03T17:25:42.550Z 15 300-319 67 67 67 120 120 120 T T F <---
74662024-02-03T17:25:42.550Z 16 320-339 68 68 68 121 121 121 F F F
74672024-02-03T17:25:42.550Z 17 340-359 66 66 66 116 116 116 T T F <---
74682024-02-03T17:25:42.550Z 18 360-379 66 66 66 116 116 116 F F F
74692024-02-03T17:25:42.550Z 19 380-399 66 66 66 118 118 118 F F F
74702024-02-03T17:25:42.550Z 20 400-419 66 66 66 118 118 118 F F F
74712024-02-03T17:25:42.550Z 21 420-439 62 62 62 109 109 109 F F F
74722024-02-03T17:25:42.550Z 22 440-459 61 61 61 108 108 108 T T F <---
74732024-02-03T17:25:42.550Z 23 460-479 64 64 64 113 113 113 F F F
74742024-02-03T17:25:42.550Z 24 480-499 66 66 66 118 118 118 F F F
74752024-02-03T17:25:42.550Z 25 500-519 66 66 66 115 115 115 F F F
74762024-02-03T17:25:42.550Z 26 520-539 66 66 66 118 118 118 T T F <---
74772024-02-03T17:25:42.551Z 27 540-559 68 68 64 124 124 113 T T F <---
74782024-02-03T17:25:42.551Z 28 560-579 65 65 65 114 114 114 F F F
74792024-02-03T17:25:42.551Z 29 580-599 65 65 65 114 114 114 T T F <---
74802024-02-03T17:25:42.551ZMax gen: 68, Max flush: 124
74812024-02-03T17:25:42.551ZError: Difference in extent metadata found!
74822024-02-03T17:25:42.551ZA Difference in extent metadata is expected here
74832024-02-03T17:25:42.551ZOn loop 34
74842024-02-03T17:25:42.551Z
74852024-02-03T17:25:42.551ZVerifying data now
74862024-02-03T17:25:47.549ZNo change after verify
74872024-02-03T17:25:47.549ZLoop: 34 Downstairs dump after verify (and repair):
74882024-02-03T17:25:47.583ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74892024-02-03T17:25:47.583Z 0 000-019 69 69 69 125 125 125 F F F
74902024-02-03T17:25:47.583Z 1 020-039 62 62 62 109 109 109 F F F
74912024-02-03T17:25:47.583Z 2 040-059 63 63 63 110 110 110 F F F
74922024-02-03T17:25:47.583Z 3 060-079 69 69 69 125 125 125 F F F
74932024-02-03T17:25:47.583Z 4 080-099 69 69 69 125 125 125 F F F
74942024-02-03T17:25:47.583Z 5 100-119 64 64 64 111 111 111 F F F
74952024-02-03T17:25:47.583Z 6 120-139 66 66 66 118 118 118 F F F
74962024-02-03T17:25:47.583Z 7 140-159 67 67 67 120 120 120 F F F
74972024-02-03T17:25:47.584Z 8 160-179 67 67 67 120 120 120 F F F
74982024-02-03T17:25:47.584Z 9 180-199 69 69 69 125 125 125 F F F
74992024-02-03T17:25:47.584Z 10 200-219 68 68 68 122 122 122 F F F
75002024-02-03T17:25:47.584Z 11 220-239 64 64 64 113 113 113 F F F
75012024-02-03T17:25:47.584Z 12 240-259 59 59 59 105 105 105 F F F
75022024-02-03T17:25:47.584Z 13 260-279 68 68 68 124 124 124 F F F
75032024-02-03T17:25:47.584Z 14 280-299 67 67 67 120 120 120 F F F
75042024-02-03T17:25:47.584Z 15 300-319 69 69 69 125 125 125 F F F
75052024-02-03T17:25:47.584Z 16 320-339 68 68 68 121 121 121 F F F
75062024-02-03T17:25:47.584Z 17 340-359 69 69 69 125 125 125 F F F
75072024-02-03T17:25:47.584Z 18 360-379 66 66 66 116 116 116 F F F
75082024-02-03T17:25:47.584Z 19 380-399 66 66 66 118 118 118 F F F
75092024-02-03T17:25:47.584Z 20 400-419 66 66 66 118 118 118 F F F
75102024-02-03T17:25:47.585Z 21 420-439 62 62 62 109 109 109 F F F
75112024-02-03T17:25:47.585Z 22 440-459 69 69 69 125 125 125 F F F
75122024-02-03T17:25:47.585Z 23 460-479 64 64 64 113 113 113 F F F
75132024-02-03T17:25:47.585Z 24 480-499 66 66 66 118 118 118 F F F
75142024-02-03T17:25:47.585Z 25 500-519 66 66 66 115 115 115 F F F
75152024-02-03T17:25:47.585Z 26 520-539 69 69 69 125 125 125 F F F
75162024-02-03T17:25:47.585Z 27 540-559 69 69 69 125 125 125 F F F
75172024-02-03T17:25:47.585Z 28 560-579 65 65 65 114 114 114 F F F
75182024-02-03T17:25:47.585Z 29 580-599 69 69 69 125 125 125 F F F
75192024-02-03T17:25:47.585ZMax gen: 69, Max flush: 125
75202024-02-03T17:25:47.585Z
75212024-02-03T17:25:47.585ZBegin loop 35
75222024-02-03T17:25:47.585ZDownstairs to restart: 1
75232024-02-03T17:25:47.632Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.776954516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412}
75242024-02-03T17:25:47.632Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.777497341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412}
75252024-02-03T17:25:47.632Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.777573031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412}
75262024-02-03T17:25:47.632Z{"msg":"Crucible 7fa5675e-a48d-489a-9a3d-e355db2d5efc has session id: b1a67585-faf1-4c69-8d25-bc8fbf3fb04a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.77763918Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75272024-02-03T17:25:47.633Z{"msg":"Crucible runtime is spawned
75282024-02-03T17:25:47.633ZUpstairs opts: Upstairs UUID: 7fa5675e-a48d-489a-9a3d-e355db2d5efc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.777798412Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75292024-02-03T17:25:47.633Z{"msg":"Crucible stats registered with UUID: 7fa5675e-a48d-489a-9a3d-e355db2d5efc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.777887321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75302024-02-03T17:25:47.633Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.777918643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412}
75312024-02-03T17:25:47.633Z{"msg":"7fa5675e-a48d-489a-9a3d-e355db2d5efc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778077691Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75322024-02-03T17:25:47.633Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.77813399Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75332024-02-03T17:25:47.633Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778160744Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75342024-02-03T17:25:47.633Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778183104Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75352024-02-03T17:25:47.633Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778278475Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75362024-02-03T17:25:47.633Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778322555Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75372024-02-03T17:25:47.633Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778356022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75382024-02-03T17:25:47.633Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778391574Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75392024-02-03T17:25:47.634Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778434507Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75402024-02-03T17:25:47.634Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778499432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75412024-02-03T17:25:47.634Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.77873152Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"io task","client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75422024-02-03T17:25:47.634Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778785296Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"io task","client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75432024-02-03T17:25:47.634Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.778880154Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"io task","client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75442024-02-03T17:25:47.634Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.780198098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75452024-02-03T17:25:47.634Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.780287893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75462024-02-03T17:25:47.634Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.780375832Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75472024-02-03T17:25:47.634Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.780479983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75482024-02-03T17:25:47.634Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.781288569Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75492024-02-03T17:25:47.634Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.781592849Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75502024-02-03T17:25:47.634Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.781652876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75512024-02-03T17:25:47.634Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.781689439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75522024-02-03T17:25:47.634Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.782126148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75532024-02-03T17:25:47.634Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.782187979Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75542024-02-03T17:25:47.634Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.782228658Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75552024-02-03T17:25:47.634Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.782542439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75562024-02-03T17:25:47.634Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.78262819Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75572024-02-03T17:25:47.634Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.782710022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75582024-02-03T17:25:47.634Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.782782646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75592024-02-03T17:25:47.634Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783419592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75602024-02-03T17:25:47.634Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783484164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75612024-02-03T17:25:47.635Z{"msg":"[0]R flush_numbers[0..12]: [125, 109, 110, 125, 125, 111, 118, 120, 120, 125, 122, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783512464Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75622024-02-03T17:25:47.635Z{"msg":"[0]R generation[0..12]: [69, 62, 63, 69, 69, 64, 66, 67, 67, 69, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783535966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75632024-02-03T17:25:47.635Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783579414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75642024-02-03T17:25:47.635Z{"msg":"[1]R flush_numbers[0..12]: [125, 109, 110, 125, 125, 111, 118, 120, 120, 125, 122, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783605562Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75652024-02-03T17:25:47.635Z{"msg":"[1]R generation[0..12]: [69, 62, 63, 69, 69, 64, 66, 67, 67, 69, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783649445Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75662024-02-03T17:25:47.635Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783675025Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75672024-02-03T17:25:47.635Z{"msg":"[2]R flush_numbers[0..12]: [125, 109, 110, 125, 125, 111, 118, 120, 120, 125, 122, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783717372Z","hostnameWait for a query_work_queue command to finish before sending IO
75682024-02-03T17:25:47.635Z":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75692024-02-03T17:25:47.635Z{"msg":"[2]R generation[0..12]: [69, 62, 63, 69, 69, 64, 66, 67, 67, 69, 68, 64]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783746966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75702024-02-03T17:25:47.635Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783770329Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75712024-02-03T17:25:47.635Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783792173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75722024-02-03T17:25:47.635Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783814148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75732024-02-03T17:25:47.635Z{"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783835402Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75742024-02-03T17:25:47.635Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783856779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"":"downstairs","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75752024-02-03T17:25:47.635Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783878689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75762024-02-03T17:25:47.635Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783898451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"0","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75772024-02-03T17:25:47.636Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783919839Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"1","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75782024-02-03T17:25:47.636Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783941677Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"client":"2","":"downstairs","session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75792024-02-03T17:25:47.636Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.783963539Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75802024-02-03T17:25:47.636Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.784018342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75812024-02-03T17:25:47.636Z{"msg":"7fa5675e-a48d-489a-9a3d-e355db2d5efc is now active with session: b1a67585-faf1-4c69-8d25-bc8fbf3fb04a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.784060058Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75822024-02-03T17:25:47.636Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.784095172Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412,"session_id":"b1a67585-faf1-4c69-8d25-bc8fbf3fb04a"}
75832024-02-03T17:25:47.636Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:46.78412222Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412}
75842024-02-03T17:25:47.636ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75852024-02-03T17:25:47.636ZLoading write count information from file "/tmp/test_repair_verify.data"
75862024-02-03T17:25:47.636ZRun Repair workload
75872024-02-03T17:25:47.636Z01/30 Flush
75882024-02-03T17:25:47.636Z02/30 Write block 475 len 28672 data: 3 3 6 7 7 6 7
75892024-02-03T17:25:47.636Z03/30 Write block 453 len 32768 data: 5 4 3 6 7 6 6 4
75902024-02-03T17:25:47.636Z04/30 Write block 321 len 36864 data: 5 7 6 6 6 7 6 7 5
75912024-02-03T17:25:47.636Z05/30 Write block 79 len 32768 data: 4 4 4 5 7 8 6 5
75922024-02-03T17:25:47.700Z06/30 Write block 138 len 40960 data: 10 8 6 6 5 3 3 4 2 2
75932024-02-03T17:25:47.700Z07/30 Write block 274 len 8192 data: 12 12
75942024-02-03T17:25:47.700Z08/30 Flush
75952024-02-03T17:25:47.700Z09/30 Write block 536 len 36864 data: 6 7 6 6 6 5 5 4 5
75962024-02-03T17:25:47.700Z10/30 Write block 363 len 28672 data: 12 11 9 6 6 6 5
75972024-02-03T17:25:47.700Z11/30 Write block 208 len 20480 data: 7 6 7 6 5
75982024-02-03T17:25:47.700Z12/30 Read block 219 len 36864
75992024-02-03T17:25:47.700Z13/30 Read block 23 len 40960
76002024-02-03T17:25:47.701Z14/30 Write block 120 len 4096 data: 6
76012024-02-03T17:25:47.701Z15/30 Flush
76022024-02-03T17:25:47.701Z16/30 Write block 112 len 40960 data: 9 10 11 9 7 8 8 7 7 7
76032024-02-03T17:25:47.701Z17/30 Write block 274 len 36864 data: 13 13 14 11 10 7 5 7 7
76042024-02-03T17:25:47.701Z18/30 Read block 100 len 16384
76052024-02-03T17:25:47.701Z19/30 Write block 224 len 4096 data: 8
76062024-02-03T17:25:47.701Z20/30 Flush
76072024-02-03T17:25:47.701Z21/30 Write block 90 len 32768 data: 6 5 5 6 5 6 5 4
76082024-02-03T17:25:47.701Z22/30 Read block 587 len 24576
76092024-02-03T17:25:47.701Z23/30 Write block 498 len 12288 data: 7 7 8
76102024-02-03T17:25:47.701Z24/30 Read block 76 len 16384
76112024-02-03T17:25:47.701Z25/30 Flush
76122024-02-03T17:25:47.702Z26/30 Write block 302 len 20480 data: 6 6 5 5 6
76132024-02-03T17:25:47.702Z27/30 Write block 534 len 8192 data: 4 6
76142024-02-03T17:25:47.702Z28/30 Read block 395 len 4096
76152024-02-03T17:25:47.702Z29/30 Read block 145 len 16384
76162024-02-03T17:25:47.702Z30/30 Write block 119 len 4096 data: 8
76172024-02-03T17:25:47.702Z----------------------------------------------------------------
76182024-02-03T17:25:47.702Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:29
76192024-02-03T17:25:47.702ZStates: Active Active Active
76202024-02-03T17:25:47.702ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76212024-02-03T17:25:47.702Z 2 Acked 1001 Write 7 Done Done Done false
76222024-02-03T17:25:47.702Z 3 Acked 1002 Write 8 Done Sent Done false
76232024-02-03T17:25:47.702Z 4 Acked 1003 Write 9 Done Sent Done false
76242024-02-03T17:25:47.703Z 5 Acked 1004 Write 8 Done Sent Done false
76252024-02-03T17:25:47.703Z 6 Acked 1005 Write 10 Done Sent Done false
76262024-02-03T17:25:47.703Z 7 Acked 1006 Write 2 Done Sent Done false
76272024-02-03T17:25:47.703Z 8 Acked 1007 Flush 0 Done Sent Done false
76282024-02-03T17:25:47.703Z 9 Acked 1008 Write 9 Done Sent Done false
76292024-02-03T17:25:47.703Z 10 Acked 1009 Write 7 Done Sent Done false
76302024-02-03T17:25:47.703Z 11 Acked 1010 Write 5 Done Sent Done false
76312024-02-03T17:25:47.703Z 12 Acked 1011 Read 9 Done Sent Done false
76322024-02-03T17:25:47.703Z 13 Acked 1012 Read 10 Done Sent Done false
76332024-02-03T17:25:47.703Z 14 Acked 1013 Write 1 Done Sent Done false
76342024-02-03T17:25:47.703Z 15 Acked 1014 Flush 0 Done Sent Done false
76352024-02-03T17:25:47.703Z 16 Acked 1015 Write 10 Done Sent Done false
76362024-02-03T17:25:47.704Z 17 Acked 1016 Write 9 Done Sent Done false
76372024-02-03T17:25:47.704Z 18 Acked 1017 Read 4 Done Sent Done false
76382024-02-03T17:25:47.704Z 19 Acked 1018 Write 1 Done Sent Done false
76392024-02-03T17:25:47.704Z 20 Acked 1019 Flush 0 Done Sent Done false
76402024-02-03T17:25:47.704Z 21 Acked 1020 Write 8 Done Sent Done false
76412024-02-03T17:25:47.704Z 22 Acked 1021 Read 6 Done Sent Done false
76422024-02-03T17:25:47.704Z 23 Acked 1022 Write 3 Done Sent Done false
76432024-02-03T17:25:47.704Z 24 Acked 1023 Read 4 Done Sent Done false
76442024-02-03T17:25:47.704Z 25 Acked 1024 Flush 0 Done Sent Done false
76452024-02-03T17:25:47.704Z 26 Acked 1025 Write 5 Done Sent Done false
76462024-02-03T17:25:47.704Z 27 Acked 1026 Write 2 Done Sent Done false
76472024-02-03T17:25:47.704Z 28 Acked 1027 Read 1 Done Sent Done false
76482024-02-03T17:25:47.704Z 29 Acked 1028 Read 4 Done Sent Done false
76492024-02-03T17:25:47.704Z 30 Acked 1029 Write 1 Sent Sent Sent false
76502024-02-03T17:25:47.705Z STATES DS:0 DS:1 DS:2 TOTAL
76512024-02-03T17:25:47.705Z New 0 0 0 0
76522024-02-03T17:25:47.705Z Sent 1 28 1 30
76532024-02-03T17:25:47.705Z Done 28 1 28 57
76542024-02-03T17:25:47.705Z Skipped 0 0 0 0
76552024-02-03T17:25:47.705Z Error 0 0 0 0
76562024-02-03T17:25:47.705ZLast Flush: 1024 1000 1024
76572024-02-03T17:25:47.705ZDownstairs last five completed: 1000
76582024-02-03T17:25:47.705ZUpstairs last five completed: 30 29 28 27 26
76592024-02-03T17:25:47.705Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:46.836779953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1412}
76602024-02-03T17:25:47.705ZWrote out file "/tmp/test_repair_verify.data"
76612024-02-03T17:25:47.705Z
76622024-02-03T17:25:47.740ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76632024-02-03T17:25:47.741ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76642024-02-03T17:25:47.741Z 0 000-019 69 69 69 125 125 125 F F F
76652024-02-03T17:25:47.741Z 1 020-039 62 62 62 109 109 109 F F F
76662024-02-03T17:25:47.741Z 2 040-059 63 63 63 110 110 110 F F F
76672024-02-03T17:25:47.741Z 3 060-079 70 69 70 127 125 127 F F F <---
76682024-02-03T17:25:47.741Z 4 080-099 70 69 70 130 125 130 F F F <---
76692024-02-03T17:25:47.741Z 5 100-119 70 64 70 129 111 129 T F T <---
76702024-02-03T17:25:47.741Z 6 120-139 70 66 70 129 118 129 F F F <---
76712024-02-03T17:25:47.741Z 7 140-159 70 67 70 127 120 127 F F F <---
76722024-02-03T17:25:47.741Z 8 160-179 67 67 67 120 120 120 F F F
76732024-02-03T17:25:47.741Z 9 180-199 69 69 69 125 125 125 F F F
76742024-02-03T17:25:47.741Z 10 200-219 70 68 70 128 122 128 F F F <---
76752024-02-03T17:25:47.741Z 11 220-239 70 64 70 129 113 129 F F F <---
76762024-02-03T17:25:47.742Z 12 240-259 59 59 59 105 105 105 F F F
76772024-02-03T17:25:47.742Z 13 260-279 70 68 70 129 124 129 F F F <---
76782024-02-03T17:25:47.742Z 14 280-299 70 67 70 129 120 129 F F F <---
76792024-02-03T17:25:47.742Z 15 300-319 69 69 69 125 125 125 T F T <---
76802024-02-03T17:25:47.742Z 16 320-339 70 68 70 127 121 127 F F F <---
76812024-02-03T17:25:47.742Z 17 340-359 69 69 69 125 125 125 F F F
76822024-02-03T17:25:47.742Z 18 360-379 70 66 70 128 116 128 F F F <---
76832024-02-03T17:25:47.742Z 19 380-399 66 66 66 118 118 118 F F F
76842024-02-03T17:25:47.742Z 20 400-419 66 66 66 118 118 118 F F F
76852024-02-03T17:25:47.742Z 21 420-439 62 62 62 109 109 109 F F F
76862024-02-03T17:25:47.742Z 22 440-459 70 69 70 127 125 127 F F F <---
76872024-02-03T17:25:47.742Z 23 460-479 70 64 70 127 113 127 F T F <---
76882024-02-03T17:25:47.742Z 24 480-499 70 66 70 130 118 130 F T F <---
76892024-02-03T17:25:47.742Z 25 500-519 70 66 70 130 115 130 F F F <---
76902024-02-03T17:25:47.742Z 26 520-539 70 69 70 128 125 128 T F T <---
76912024-02-03T17:25:47.742Z 27 540-559 70 69 70 128 125 128 F F F <---
76922024-02-03T17:25:47.743Z 28 560-579 65 65 65 114 114 114 F F F
76932024-02-03T17:25:47.743Z 29 580-599 69 69 69 125 125 125 F F F
76942024-02-03T17:25:47.743ZMax gen: 70, Max flush: 130
76952024-02-03T17:25:47.743ZError: Difference in extent metadata found!
76962024-02-03T17:25:47.743ZA Difference in extent metadata is expected here
76972024-02-03T17:25:47.743ZOn loop 35
76982024-02-03T17:25:47.743Z
76992024-02-03T17:25:47.743ZVerifying data now
77002024-02-03T17:26:09.997ZNo change after verify
77012024-02-03T17:26:09.997ZLoop: 35 Downstairs dump after verify (and repair):
77022024-02-03T17:26:10.066ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77032024-02-03T17:26:10.066Z 0 000-019 69 69 69 125 125 125 F F F
77042024-02-03T17:26:10.067Z 1 020-039 62 62 62 109 109 109 F F F
77052024-02-03T17:26:10.067Z 2 040-059 63 63 63 110 110 110 F F F
77062024-02-03T17:26:10.067Z 3 060-079 70 70 70 127 127 127 F F F
77072024-02-03T17:26:10.067Z 4 080-099 70 70 70 130 130 130 F F F
77082024-02-03T17:26:10.067Z 5 100-119 71 71 71 131 131 131 F F F
77092024-02-03T17:26:10.067Z 6 120-139 70 70 70 129 129 129 F F F
77102024-02-03T17:26:10.067Z 7 140-159 70 70 70 127 127 127 F F F
77112024-02-03T17:26:10.067Z 8 160-179 67 67 67 120 120 120 F F F
77122024-02-03T17:26:10.067Z 9 180-199 69 69 69 125 125 125 F F F
77132024-02-03T17:26:10.067Z 10 200-219 70 70 70 128 128 128 F F F
77142024-02-03T17:26:10.067Z 11 220-239 70 70 70 129 129 129 F F F
77152024-02-03T17:26:10.067Z 12 240-259 59 59 59 105 105 105 F F F
77162024-02-03T17:26:10.067Z 13 260-279 70 70 70 129 129 129 F F F
77172024-02-03T17:26:10.067Z 14 280-299 70 70 70 129 129 129 F F F
77182024-02-03T17:26:10.067Z 15 300-319 71 71 71 131 131 131 F F F
77192024-02-03T17:26:10.067Z 16 320-339 70 70 70 127 127 127 F F F
77202024-02-03T17:26:10.067Z 17 340-359 69 69 69 125 125 125 F F F
77212024-02-03T17:26:10.067Z 18 360-379 70 70 70 128 128 128 F F F
77222024-02-03T17:26:10.067Z 19 380-399 66 66 66 118 118 118 F F F
77232024-02-03T17:26:10.067Z 20 400-419 66 66 66 118 118 118 F F F
77242024-02-03T17:26:10.067Z 21 420-439 62 62 62 109 109 109 F F F
77252024-02-03T17:26:10.068Z 22 440-459 70 70 70 127 127 127 F F F
77262024-02-03T17:26:10.068Z 23 460-479 70 70 70 127 127 127 F F F
77272024-02-03T17:26:10.068Z 24 480-499 70 70 70 130 130 130 F F F
77282024-02-03T17:26:10.068Z 25 500-519 70 70 70 130 130 130 F F F
77292024-02-03T17:26:10.068Z 26 520-539 71 71 71 131 131 131 F F F
77302024-02-03T17:26:10.068Z 27 540-559 70 70 70 128 128 128 F F F
77312024-02-03T17:26:10.068Z 28 560-579 65 65 65 114 114 114 F F F
77322024-02-03T17:26:10.068Z 29 580-599 69 69 69 125 125 125 F F F
77332024-02-03T17:26:10.068ZMax gen: 71, Max flush: 131
77342024-02-03T17:26:10.068Z
77352024-02-03T17:26:10.068ZBegin loop 36
77362024-02-03T17:26:10.068ZDownstairs to restart: 2
77372024-02-03T17:26:10.068Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.470933495Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420}
77382024-02-03T17:26:10.068Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.471491451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420}
77392024-02-03T17:26:10.068Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.471557411Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420}
77402024-02-03T17:26:10.068ZCrucible runtime is spawned
77412024-02-03T17:26:10.068Z{"msg":"Crucible 58c0b7dc-47e4-4ed9-9efe-d01c0d9f1847 has session id: 24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.4716606Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77422024-02-03T17:26:10.068Z{"msg":"Upstairs opts: Upstairs UUID: 58c0b7dc-47e4-4ed9-9efe-d01c0d9f1847, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.471728479Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77432024-02-03T17:26:10.068Z{"msg":"Crucible stats registered with UUID: 58c0b7dc-47e4-4ed9-9efe-d01c0d9f1847","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.471785782Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77442024-02-03T17:26:10.068Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.471826193Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420}
77452024-02-03T17:26:10.069Z{"msg":"58c0b7dc-47e4-4ed9-9efe-d01c0d9f1847 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.471912656Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77462024-02-03T17:26:10.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.471991824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77472024-02-03T17:26:10.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472068004Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77482024-02-03T17:26:10.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472113577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77492024-02-03T17:26:10.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472187408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77502024-02-03T17:26:10.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472217784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77512024-02-03T17:26:10.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472269234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77522024-02-03T17:26:10.069Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472320589Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77532024-02-03T17:26:10.069Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472348073Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77542024-02-03T17:26:10.069Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.47237536Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77552024-02-03T17:26:10.069Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472643398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"io task","client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77562024-02-03T17:26:10.069Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472698887Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"io task","client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77572024-02-03T17:26:10.069Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.472792592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"io task","client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77582024-02-03T17:26:10.069Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.474184053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77592024-02-03T17:26:10.069Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.474272685Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77602024-02-03T17:26:10.069Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.474374467Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77612024-02-03T17:26:10.069Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.474419141Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77622024-02-03T17:26:10.069Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.474681996Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77632024-02-03T17:26:10.069Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.474769779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77642024-02-03T17:26:10.069Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.475137277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77652024-02-03T17:26:10.069Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.475431662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77662024-02-03T17:26:10.069Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.475838079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77672024-02-03T17:26:10.069Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476085287Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77682024-02-03T17:26:10.070Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476149127Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77692024-02-03T17:26:10.070Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476191079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77702024-02-03T17:26:10.070Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476247126Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77712024-02-03T17:26:10.070Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476273674Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77722024-02-03T17:26:10.070Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476315045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77732024-02-03T17:26:10.070Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476682363Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77742024-02-03T17:26:10.070Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476738065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77752024-02-03T17:26:10.070Z{"msg":"[0]R flush_numbers[0..12]: [125, 109, 110, 127, 130, 131, 129, 127, 120, 125, 128, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476764868Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77762024-02-03T17:26:10.070Z{"msg":"[0]R generation[0..12]: [69, 62, 63, 70, 70, 71, 70, 70, 67, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476788177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77772024-02-03T17:26:10.070Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476837483Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77782024-02-03T17:26:10.070Z{"msg":"[1]R flush_numbers[0..12]: [125, 109, 110, 127, 130, 131, 129, 127, 120, 125, 128, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476886009Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77792024-02-03T17:26:10.070Z{"msg":"[1]R generation[0..12]: [69, 62, 63, 70, 70, 71, 70, 70, 67, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476911163Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77802024-02-03T17:26:10.070Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476953694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77812024-02-03T17:26:10.070Z{"msg":"[2]R flush_numbers[0..12]: [125, 109, 110, 127, 130, 131, 129, 127, 120, 125, 128, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.476979441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77822024-02-03T17:26:10.070Z{"msg":"[2]R generation[0..12]: [69, 62, 63, 70, 70, 71, 70, 70, 67, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477003448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77832024-02-03T17:26:10.070Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477026706Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77842024-02-03T17:26:10.070Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
77852024-02-03T17:26:10.070Z,"time":"2024-02-03T17:25:53.477067303Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77862024-02-03T17:26:10.071Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477092964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77872024-02-03T17:26:10.071Z{"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.47713935Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77882024-02-03T17:26:10.071Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477163281Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"":"downstairs","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77892024-02-03T17:26:10.071Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477204736Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77902024-02-03T17:26:10.071Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.47722675Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"0","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77912024-02-03T17:26:10.071Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477275433Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"1","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77922024-02-03T17:26:10.071Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477299147Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"client":"2","":"downstairs","session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77932024-02-03T17:26:10.071Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477343914Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77942024-02-03T17:26:10.071Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.47736637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77952024-02-03T17:26:10.071Z{"msg":"58c0b7dc-47e4-4ed9-9efe-d01c0d9f1847 is now active with session: 24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477388907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77962024-02-03T17:26:10.071Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477430755Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420,"session_id":"24d8c38e-7c0f-4962-aa70-d3fb52e7d2f4"}
77972024-02-03T17:26:10.071Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:53.477453173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420}
77982024-02-03T17:26:10.071ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
77992024-02-03T17:26:10.071ZLoading write count information from file "/tmp/test_repair_verify.data"
78002024-02-03T17:26:10.071ZRun Repair workload
78012024-02-03T17:26:10.071Z01/30 Write block 423 len 32768 data: 5 5 5 4 3 2 2 3
78022024-02-03T17:26:10.087Z02/30 Write block 282 len 24576 data: 8 7 8 6 5 7
78032024-02-03T17:26:10.087Z03/30 Write block 580 len 32768 data: 9 8 8 7 7 5 6 5
78042024-02-03T17:26:10.087Z04/30 Read block 591 len 16384
78052024-02-03T17:26:10.087Z05/30 Write block 312 len 36864 data: 2 4 3 3 3 2 2 6 4
78062024-02-03T17:26:10.087Z06/30 Read block 21 len 16384
78072024-02-03T17:26:10.087Z07/30 Read block 205 len 20480
78082024-02-03T17:26:10.087Z08/30 Flush
78092024-02-03T17:26:10.087Z09/30 Write block 522 len 28672 data: 3 4 4 5 7 8 7
78102024-02-03T17:26:10.087Z10/30 Write block 278 len 40960 data: 11 8 6 8 9 8 9 7 6 8
78112024-02-03T17:26:10.087Z11/30 Write block 457 len 4096 data: 8
78122024-02-03T17:26:10.087Z12/30 Read block 438 len 12288
78132024-02-03T17:26:10.088Z13/30 Write block 90 len 20480 data: 7 6 6 7 6
78142024-02-03T17:26:10.088Z14/30 Read block 577 len 20480
78152024-02-03T17:26:10.088Z15/30 Read block 433 len 24576
78162024-02-03T17:26:10.088Z16/30 Read block 169 len 20480
78172024-02-03T17:26:10.088Z17/30 Write block 159 len 20480 data: 6 6 7 8 6
78182024-02-03T17:26:10.088Z18/30 Write block 405 len 40960 data: 9 8 7 5 6 6 7 6 6 5
78192024-02-03T17:26:10.088Z19/30 Flush
78202024-02-03T17:26:10.088Z20/30 Write block 94 len 4096 data: 7
78212024-02-03T17:26:10.088Z21/30 Read block 260 len 12288
78222024-02-03T17:26:10.088Z22/30 Read block 215 len 4096
78232024-02-03T17:26:10.088Z23/30 Write block 351 len 36864 data: 5 5 5 5 6 9 9 9 10
78242024-02-03T17:26:10.088Z24/30 Write block 358 len 16384 data: 10 11 10 11
78252024-02-03T17:26:10.088Z25/30 Read block 338 len 28672
78262024-02-03T17:26:10.088Z26/30 Write block 436 len 24576 data: 3 3 3 4 6 6
78272024-02-03T17:26:10.088Z27/30 Read block 426 len 20480
78282024-02-03T17:26:10.088Z28/30 Write block 342 len 8192 data: 5 5
78292024-02-03T17:26:10.088Z29/30 Read block 341 len 40960
78302024-02-03T17:26:10.088Z30/30 Write block 107 len 24576 data: 7 8 8 8 9 10
78312024-02-03T17:26:10.088Z----------------------------------------------------------------
78322024-02-03T17:26:10.088Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78332024-02-03T17:26:10.088ZStates: Active Active Active
78342024-02-03T17:26:10.088ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78352024-02-03T17:26:10.088Z 1 Acked 1000 Write 8 Done Done Sent false
78362024-02-03T17:26:10.088Z 2 Acked 1001 Write 6 Done Done Sent false
78372024-02-03T17:26:10.089Z 3 Acked 1002 Write 8 Done Done Sent false
78382024-02-03T17:26:10.089Z 4 Acked 1003 Read 4 Done Done Sent false
78392024-02-03T17:26:10.089Z 5 Acked 1004 Write 9 Done Done Sent false
78402024-02-03T17:26:10.089Z 6 Acked 1005 Read 4 Done Done Sent false
78412024-02-03T17:26:10.089Z 7 Acked 1006 Read 5 Done Done Sent false
78422024-02-03T17:26:10.089Z 8 Acked 1007 Flush 0 Done Done Sent false
78432024-02-03T17:26:10.089Z 9 Acked 1008 Write 7 Done Done Sent false
78442024-02-03T17:26:10.089Z 10 Acked 1009 Write 10 Done Done Sent false
78452024-02-03T17:26:10.089Z 11 Acked 1010 Write 1 Done Done Sent false
78462024-02-03T17:26:10.089Z 12 Acked 1011 Read 3 Done Done Sent false
78472024-02-03T17:26:10.089Z 13 Acked 1012 Write 5 Done Done Sent false
78482024-02-03T17:26:10.089Z 14 Acked 1013 Read 5 Done Done Sent false
78492024-02-03T17:26:10.089Z 15 Acked 1014 Read 6 Done Done Sent false
78502024-02-03T17:26:10.089Z 16 Acked 1015 Read 5 Done Done Sent false
78512024-02-03T17:26:10.089Z 17 Acked 1016 Write 5 Done Done Sent false
78522024-02-03T17:26:10.089Z 18 Acked 1017 Write 10 Done Done Sent false
78532024-02-03T17:26:10.089Z 19 Acked 1018 Flush 0 Done Done Sent false
78542024-02-03T17:26:10.089Z 20 Acked 1019 Write 1 Done Done Sent false
78552024-02-03T17:26:10.089Z 21 Acked 1020 Read 3 Done Done Sent false
78562024-02-03T17:26:10.089Z 22 Acked 1021 Read 1 Done Done Sent false
78572024-02-03T17:26:10.089Z 23 Acked 1022 Write 9 Done Done Sent false
78582024-02-03T17:26:10.090Z 24 Acked 1023 Write 4 Done Done Sent false
78592024-02-03T17:26:10.090Z 25 Acked 1024 Read 7 Done Done Sent false
78602024-02-03T17:26:10.090Z 26 Acked 1025 Write 6 Done Done Sent false
78612024-02-03T17:26:10.090Z 27 Acked 1026 Read 5 Done Done Sent false
78622024-02-03T17:26:10.090Z 28 Acked 1027 Write 2 Done Done Sent false
78632024-02-03T17:26:10.090Z 29 Acked 1028 Read 10 Done Done Sent false
78642024-02-03T17:26:10.090Z 30 Acked 1029 Write 6 Sent Sent Sent false
78652024-02-03T17:26:10.090Z STATES DS:0 DS:1 DS:2 TOTAL
78662024-02-03T17:26:10.090Z New 0 0 0 0
78672024-02-03T17:26:10.090Z Sent 1 1 30 32
78682024-02-03T17:26:10.090Z Done 29 29 0 58
78692024-02-03T17:26:10.090Z Skipped 0 0 0 0
78702024-02-03T17:26:10.090Z Error 0 0 0 0
78712024-02-03T17:26:10.090ZLast Flush: 1018 1018 0
78722024-02-03T17:26:10.090ZDownstairs last five completed:
78732024-02-03T17:26:10.090ZUpstairs last five completed: 30 29 28 27 26
78742024-02-03T17:26:10.090Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:53.526890537Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1420}
78752024-02-03T17:26:10.090ZWrote out file "/tmp/test_repair_verify.data"
78762024-02-03T17:26:10.090Z
78772024-02-03T17:26:10.090ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78782024-02-03T17:26:10.090ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78792024-02-03T17:26:10.090Z 0 000-019 69 69 69 125 125 125 F F F
78802024-02-03T17:26:10.090Z 1 020-039 62 62 62 109 109 109 F F F
78812024-02-03T17:26:10.090Z 2 040-059 63 63 63 110 110 110 F F F
78822024-02-03T17:26:10.091Z 3 060-079 70 70 70 127 127 127 F F F
78832024-02-03T17:26:10.091Z 4 080-099 72 72 70 133 133 130 T T F <---
78842024-02-03T17:26:10.091Z 5 100-119 71 71 71 131 131 131 T T F <---
78852024-02-03T17:26:10.091Z 6 120-139 70 70 70 129 129 129 F F F
78862024-02-03T17:26:10.091Z 7 140-159 72 72 70 133 133 127 F F F <---
78872024-02-03T17:26:10.091Z 8 160-179 72 72 67 133 133 120 F F F <---
78882024-02-03T17:26:10.091Z 9 180-199 69 69 69 125 125 125 F F F
78892024-02-03T17:26:10.091Z 10 200-219 70 70 70 128 128 128 F F F
78902024-02-03T17:26:10.091Z 11 220-239 70 70 70 129 129 129 F F F
78912024-02-03T17:26:10.091Z 12 240-259 59 59 59 105 105 105 F F F
78922024-02-03T17:26:10.091Z 13 260-279 72 72 70 133 133 129 F F F <---
78932024-02-03T17:26:10.091Z 14 280-299 72 72 70 133 133 129 F F F <---
78942024-02-03T17:26:10.091Z 15 300-319 72 72 71 132 132 131 F F F <---
78952024-02-03T17:26:10.091Z 16 320-339 72 72 70 132 132 127 F F F <---
78962024-02-03T17:26:10.091Z 17 340-359 69 69 69 125 125 125 T T F <---
78972024-02-03T17:26:10.091Z 18 360-379 70 70 70 128 128 128 T T F <---
78982024-02-03T17:26:10.091Z 19 380-399 66 66 66 118 118 118 F F F
78992024-02-03T17:26:10.091Z 20 400-419 72 72 66 133 133 118 F F F <---
79002024-02-03T17:26:10.091Z 21 420-439 72 72 62 132 132 109 T T F <---
79012024-02-03T17:26:10.091Z 22 440-459 72 72 70 133 133 127 T T F <---
79022024-02-03T17:26:10.102Z 23 460-479 70 70 70 127 127 127 F F F
79032024-02-03T17:26:10.102Z 24 480-499 70 70 70 130 130 130 F F F
79042024-02-03T17:26:10.102Z 25 500-519 70 70 70 130 130 130 F F F
79052024-02-03T17:26:10.103Z 26 520-539 72 72 71 133 133 131 F F F <---
79062024-02-03T17:26:10.103Z 27 540-559 70 70 70 128 128 128 F F F
79072024-02-03T17:26:10.103Z 28 560-579 65 65 65 114 114 114 F F F
79082024-02-03T17:26:10.103Z 29 580-599 72 72 69 132 132 125 F F F <---
79092024-02-03T17:26:10.103ZMax gen: 72, Max flush: 133
79102024-02-03T17:26:10.103ZError: Difference in extent metadata found!
79112024-02-03T17:26:10.103ZA Difference in extent metadata is expected here
79122024-02-03T17:26:10.103ZOn loop 36
79132024-02-03T17:26:10.103Z
79142024-02-03T17:26:10.103ZVerifying data now
79152024-02-03T17:26:10.103ZNo change after verify
79162024-02-03T17:26:10.103ZLoop: 36 Downstairs dump after verify (and repair):
79172024-02-03T17:26:10.103ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79182024-02-03T17:26:10.104Z 0 000-019 69 69 69 125 125 125 F F F
79192024-02-03T17:26:10.104Z 1 020-039 62 62 62 109 109 109 F F F
79202024-02-03T17:26:10.104Z 2 040-059 63 63 63 110 110 110 F F F
79212024-02-03T17:26:10.104Z 3 060-079 70 70 70 127 127 127 F F F
79222024-02-03T17:26:10.104Z 4 080-099 73 73 73 134 134 134 F F F
79232024-02-03T17:26:10.104Z 5 100-119 73 73 73 134 134 134 F F F
79242024-02-03T17:26:10.104Z 6 120-139 70 70 70 129 129 129 F F F
79252024-02-03T17:26:10.104Z 7 140-159 72 72 72 133 133 133 F F F
79262024-02-03T17:26:10.104Z 8 160-179 72 72 72 133 133 133 F F F
79272024-02-03T17:26:10.104Z 9 180-199 69 69 69 125 125 125 F F F
79282024-02-03T17:26:10.104Z 10 200-219 70 70 70 128 128 128 F F F
79292024-02-03T17:26:10.104Z 11 220-239 70 70 70 129 129 129 F F F
79302024-02-03T17:26:10.104Z 12 240-259 59 59 59 105 105 105 F F F
79312024-02-03T17:26:10.105Z 13 260-279 72 72 72 133 133 133 F F F
79322024-02-03T17:26:10.105Z 14 280-299 72 72 72 133 133 133 F F F
79332024-02-03T17:26:10.105Z 15 300-319 72 72 72 132 132 132 F F F
79342024-02-03T17:26:10.105Z 16 320-339 72 72 72 132 132 132 F F F
79352024-02-03T17:26:10.105Z 17 340-359 73 73 73 134 134 134 F F F
79362024-02-03T17:26:10.105Z 18 360-379 73 73 73 134 134 134 F F F
79372024-02-03T17:26:10.105Z 19 380-399 66 66 66 118 118 118 F F F
79382024-02-03T17:26:10.105Z 20 400-419 72 72 72 133 133 133 F F F
79392024-02-03T17:26:10.105Z 21 420-439 73 73 73 134 134 134 F F F
79402024-02-03T17:26:10.105Z 22 440-459 73 73 73 134 134 134 F F F
79412024-02-03T17:26:10.105Z 23 460-479 70 70 70 127 127 127 F F F
79422024-02-03T17:26:10.105Z 24 480-499 70 70 70 130 130 130 F F F
79432024-02-03T17:26:10.105Z 25 500-519 70 70 70 130 130 130 F F F
79442024-02-03T17:26:10.106Z 26 520-539 72 72 72 133 133 133 F F F
79452024-02-03T17:26:10.106Z 27 540-559 70 70 70 128 128 128 F F F
79462024-02-03T17:26:10.106Z 28 560-579 65 65 65 114 114 114 F F F
79472024-02-03T17:26:10.106Z 29 580-599 72 72 72 132 132 132 F F F
79482024-02-03T17:26:10.106ZMax gen: 73, Max flush: 134
79492024-02-03T17:26:10.106Z
79502024-02-03T17:26:10.106ZBegin loop 37
79512024-02-03T17:26:10.106ZDownstairs to restart: 0
79522024-02-03T17:26:10.106Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.948695501Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428}
79532024-02-03T17:26:10.106Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949345866Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428}
79542024-02-03T17:26:10.106Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949392866Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428}
79552024-02-03T17:26:10.106Z{"msg":"Crucible e468849b-1011-4c4b-9d37-d7bf24370e48 has session id: b09dde5a-5c9a-40e3-a551-3e8332672ce9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949458363Z","hostname":"Crucible runtime is spawned
79562024-02-03T17:26:10.107Zip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79572024-02-03T17:26:10.107Z{"msg":"Upstairs opts: Upstairs UUID: e468849b-1011-4c4b-9d37-d7bf24370e48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949657509Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79582024-02-03T17:26:10.107Z{"msg":"Crucible stats registered with UUID: e468849b-1011-4c4b-9d37-d7bf24370e48","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949712593Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79592024-02-03T17:26:10.107Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949737085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428}
79602024-02-03T17:26:10.107Z{"msg":"e468849b-1011-4c4b-9d37-d7bf24370e48 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949756102Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79612024-02-03T17:26:10.107Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949777529Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79622024-02-03T17:26:10.107Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949800619Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79632024-02-03T17:26:10.107Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949823129Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79642024-02-03T17:26:10.107Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949862537Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79652024-02-03T17:26:10.107Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949908758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79662024-02-03T17:26:10.107Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.949973838Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79672024-02-03T17:26:10.107Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.95004828Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79682024-02-03T17:26:10.107Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.950116554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79692024-02-03T17:26:10.108Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.950299673Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79702024-02-03T17:26:10.108Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.950387471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"io task","client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79712024-02-03T17:26:10.108Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.950445351Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"io task","client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79722024-02-03T17:26:10.108Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.950531801Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"io task","client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79732024-02-03T17:26:10.108Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.951931694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79742024-02-03T17:26:10.108Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.952045319Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79752024-02-03T17:26:10.108Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.952121953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79762024-02-03T17:26:10.108Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.952199286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79772024-02-03T17:26:10.108Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.9522747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79782024-02-03T17:26:10.108Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.952367284Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79792024-02-03T17:26:10.108Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.952945343Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79802024-02-03T17:26:10.108Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.953096015Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79812024-02-03T17:26:10.108Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.953500271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79822024-02-03T17:26:10.109Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.953944218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79832024-02-03T17:26:10.109Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954020541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79842024-02-03T17:26:10.109Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954094502Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79852024-02-03T17:26:10.109Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954169429Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79862024-02-03T17:26:10.109Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954237768Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79872024-02-03T17:26:10.109Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954264551Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79882024-02-03T17:26:10.109Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954305145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79892024-02-03T17:26:10.110Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954347624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79902024-02-03T17:26:10.110Z{"msg":"[0]R flush_numbers[0..12]: [125, 109, 110, 127, 134, 134, 129, 133, 133, 125, 128, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954391143Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79912024-02-03T17:26:10.110Z{"msg":"[0]R generation[0..12]: [69, 62, 63, 70, 73, 73, 70, 72, 72, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954434847Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79922024-02-03T17:26:10.110Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954478736Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79932024-02-03T17:26:10.110Z{"msg":"[1]R flush_numbers[0..12]: [125, 109, 110, 127, 134, 134, 129, 133, 133, 125, 128, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.95452235Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79942024-02-03T17:26:10.110Z{"msg":"[1]R generation[0..12]: [69, 62, 63, 70, 73, 73, 70, 72, 72, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954565813Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79952024-02-03T17:26:10.110Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
79962024-02-03T17:26:10.110Z,"time":"2024-02-03T17:25:58.954627766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79972024-02-03T17:26:10.110Z{"msg":"[2]R flush_numbers[0..12]: [125, 109, 110, 127, 134, 134, 129, 133, 133, 125, 128, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954737615Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79982024-02-03T17:26:10.110Z{"msg":"[2]R generation[0..12]: [69, 62, 63, 70, 73, 73, 70, 72, 72, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954787191Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
79992024-02-03T17:26:10.110Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954831656Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80002024-02-03T17:26:10.110Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954874693Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80012024-02-03T17:26:10.111Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.954926949Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80022024-02-03T17:26:10.129Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955011897Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80032024-02-03T17:26:10.129Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955055343Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"":"downstairs","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80042024-02-03T17:26:10.129Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955090609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80052024-02-03T17:26:10.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955154043Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"0","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80062024-02-03T17:26:10.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955217031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"1","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80072024-02-03T17:26:10.129Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955276453Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"client":"2","":"downstairs","session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80082024-02-03T17:26:10.130Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80092024-02-03T17:26:10.130Z,"time":"2024-02-03T17:25:58.955342025Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80102024-02-03T17:26:10.130Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955409903Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80112024-02-03T17:26:10.130Z{"msg":"e468849b-1011-4c4b-9d37-d7bf24370e48 is now active with session: b09dde5a-5c9a-40e3-a551-3e8332672ce9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955481268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80122024-02-03T17:26:10.130Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955542333Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428,"session_id":"b09dde5a-5c9a-40e3-a551-3e8332672ce9"}
80132024-02-03T17:26:10.130Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:25:58.955599604Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428}
80142024-02-03T17:26:10.130ZLoading write count information from file "/tmp/test_repair_verify.data"
80152024-02-03T17:26:10.130ZRun Repair workload
80162024-02-03T17:26:10.130ZError: Difference in extent metadata found!
80172024-02-03T17:26:10.130Z01/30 Write block 478 len 16384 data: 8 8 7 8
80182024-02-03T17:26:10.130Z02/30 Read block 482 len 40960
80192024-02-03T17:26:10.130Z03/30 Flush
80202024-02-03T17:26:10.130Z04/30 Write block 464 len 16384 data: 6 7 5 6
80212024-02-03T17:26:10.130Z05/30 Write block 314 len 32768 data: 4 4 4 3 3 7 5 6
80222024-02-03T17:26:10.130Z06/30 Write block 305 len 28672 data: 6 7 4 5 5 5 4
80232024-02-03T17:26:10.130Z07/30 Read block 18 len 40960
80242024-02-03T17:26:10.130Z08/30 Flush
80252024-02-03T17:26:10.130Z09/30 Write block 132 len 28672 data: 9 6 8 8 10 9 11
80262024-02-03T17:26:10.130Z10/30 Read block 425 len 8192
80272024-02-03T17:26:10.130Z11/30 Read block 38 len 16384
80282024-02-03T17:26:10.130Z12/30 Flush
80292024-02-03T17:26:10.130Z13/30 Write block 73 len 4096 data: 3
80302024-02-03T17:26:10.130Z14/30 Write block 596 len 12288 data: 7 7 6
80312024-02-03T17:26:10.131Z15/30 Write block 427 len 8192 data: 4 3
80322024-02-03T17:26:10.131Z16/30 Flush
80332024-02-03T17:26:10.131Z17/30 Write block 461 len 28672 data: 4 6 6 7 8 6 7
80342024-02-03T17:26:10.131Z18/30 Flush
80352024-02-03T17:26:10.131Z19/30 Write block 208 len 12288 data: 8 7 8
80362024-02-03T17:26:10.131Z20/30 Write block 458 len 32768 data: 7 7 5 5 7 7 8 9
80372024-02-03T17:26:10.131Z21/30 Write block 400 len 4096 data: 8
80382024-02-03T17:26:10.131Z22/30 Read block 129 len 12288
80392024-02-03T17:26:10.131Z23/30 Write block 192 len 4096 data: 4
80402024-02-03T17:26:10.131Z24/30 Write block 476 len 8192 data: 4 7
80412024-02-03T17:26:10.131Z25/30 Read block 439 len 36864
80422024-02-03T17:26:10.131Z26/30 Read block 45 len 36864
80432024-02-03T17:26:10.131Z27/30 Write block 434 len 32768 data: 4 4 4 4 4 5 7 7
80442024-02-03T17:26:10.131Z28/30 Read block 137 len 12288
80452024-02-03T17:26:10.131Z29/30 Write block 169 len 40960 data: 8 8 7 6 5 5 5 5 7 6
80462024-02-03T17:26:10.131Z30/30 Write block 24 len 36864 data: 6 7 5 5 6 5 3 5 4
80472024-02-03T17:26:10.131Z----------------------------------------------------------------
80482024-02-03T17:26:10.131Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:27
80492024-02-03T17:26:10.131ZStates: Active Active Active
80502024-02-03T17:26:10.131ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80512024-02-03T17:26:10.131Z 4 Acked 1003 Write 4 Done Done Done false
80522024-02-03T17:26:10.131Z 5 Acked 1004 Write 8 Done Done Done false
80532024-02-03T17:26:10.131Z 6 Acked 1005 Write 7 Sent Done Done false
80542024-02-03T17:26:10.132Z 7 Acked 1006 Read 10 Sent Done Done false
80552024-02-03T17:26:10.132Z 8 Acked 1007 Flush 0 Sent Done Done false
80562024-02-03T17:26:10.132Z 9 Acked 1008 Write 7 Sent Done Done false
80572024-02-03T17:26:10.132Z 10 Acked 1009 Read 2 Sent Done Done false
80582024-02-03T17:26:10.132Z 11 Acked 1010 Read 4 Sent Done Done false
80592024-02-03T17:26:10.132Z 12 Acked 1011 Flush 0 Sent Done Done false
80602024-02-03T17:26:10.132Z 13 Acked 1012 Write 1 Sent Done Done false
80612024-02-03T17:26:10.132Z 14 Acked 1013 Write 3 Sent Done Done false
80622024-02-03T17:26:10.132Z 15 Acked 1014 Write 2 Sent Done Done false
80632024-02-03T17:26:10.132Z 16 Acked 1015 Flush 0 Sent Done Done false
80642024-02-03T17:26:10.132Z 17 Acked 1016 Write 7 Sent Done Done false
80652024-02-03T17:26:10.132Z 18 Acked 1017 Flush 0 Sent Done Done false
80662024-02-03T17:26:10.132Z 19 Acked 1018 Write 3 Sent Done Done false
80672024-02-03T17:26:10.132Z 20 Acked 1019 Write 8 Sent Done Done false
80682024-02-03T17:26:10.132Z 21 Acked 1020 Write 1 Sent Done Done false
80692024-02-03T17:26:10.132Z 22 Acked 1021 Read 3 Sent Done Done false
80702024-02-03T17:26:10.132Z 23 Acked 1022 Write 1 Sent Done Done false
80712024-02-03T17:26:10.132Z 24 Acked 1023 Write 2 Sent Done Done false
80722024-02-03T17:26:10.132Z 25 Acked 1024 Read 9 Sent Done Done false
80732024-02-03T17:26:10.132Z 26 Acked 1025 Read 9 Sent Done Done false
80742024-02-03T17:26:10.132Z 27 Acked 1026 Write 8 Sent Done Done false
80752024-02-03T17:26:10.132Z 28 Acked 1027 Read 3 Sent Done Done false
80762024-02-03T17:26:10.132Z 29 Acked 1028 Write 10 Sent Done Done false
80772024-02-03T17:26:10.132Z 30 Acked 1029 Write 9 Sent Sent Sent false
80782024-02-03T17:26:10.132Z STATES DS:0 DS:1 DS:2 TOTAL
80792024-02-03T17:26:10.133Z New 0 0 0 0
80802024-02-03T17:26:10.133Z Sent 25 1 1 27
80812024-02-03T17:26:10.133Z Done 2 26 26 54
80822024-02-03T17:26:10.133Z Skipped 0 0 0 0
80832024-02-03T17:26:10.133Z Error 0 0 0 0
80842024-02-03T17:26:10.133ZLast Flush: 1002 1017 1017
80852024-02-03T17:26:10.133ZDownstairs last five completed: 1002 1001 1000
80862024-02-03T17:26:10.133ZUpstairs last five completed: 30 29 28 27 26
80872024-02-03T17:26:10.133Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:25:59.007586257Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1428}
80882024-02-03T17:26:10.133ZWrote out file "/tmp/test_repair_verify.data"
80892024-02-03T17:26:10.133Z
80902024-02-03T17:26:10.133ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80912024-02-03T17:26:10.133ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80922024-02-03T17:26:10.133Z 0 000-019 69 69 69 125 125 125 F F F
80932024-02-03T17:26:10.133Z 1 020-039 62 62 62 109 109 109 F T T <---
80942024-02-03T17:26:10.133Z 2 040-059 63 63 63 110 110 110 F F F
80952024-02-03T17:26:10.133Z 3 060-079 70 74 74 127 138 138 F F F <---
80962024-02-03T17:26:10.133Z 4 080-099 73 73 73 134 134 134 F F F
80972024-02-03T17:26:10.133Z 5 100-119 73 73 73 134 134 134 F F F
80982024-02-03T17:26:10.133Z 6 120-139 70 74 74 129 137 137 F F F <---
80992024-02-03T17:26:10.133Z 7 140-159 72 72 72 133 133 133 F F F
81002024-02-03T17:26:10.133Z 8 160-179 72 72 72 133 133 133 F T T <---
81012024-02-03T17:26:10.133Z 9 180-199 69 69 69 125 125 125 F T T <---
81022024-02-03T17:26:10.148Z 10 200-219 70 70 70 128 128 128 F T T <---
81032024-02-03T17:26:10.148Z 11 220-239 70 70 70 129 129 129 F F F
81042024-02-03T17:26:10.148Z 12 240-259 59 59 59 105 105 105 F F F
81052024-02-03T17:26:10.148Z 13 260-279 72 72 72 133 133 133 F F F
81062024-02-03T17:26:10.148Z 14 280-299 72 72 72 133 133 133 F F F
81072024-02-03T17:26:10.148Z 15 300-319 72 74 74 132 136 136 T F F <---
81082024-02-03T17:26:10.148Z 16 320-339 72 74 74 132 136 136 T F F <---
81092024-02-03T17:26:10.148Z 17 340-359 73 73 73 134 134 134 F F F
81102024-02-03T17:26:10.148Z 18 360-379 73 73 73 134 134 134 F F F
81112024-02-03T17:26:10.148Z 19 380-399 66 66 66 118 118 118 F F F
81122024-02-03T17:26:10.148Z 20 400-419 72 72 72 133 133 133 F T T <---
81132024-02-03T17:26:10.148Z 21 420-439 73 74 74 134 138 138 F T T <---
81142024-02-03T17:26:10.148Z 22 440-459 73 73 73 134 134 134 F T T <---
81152024-02-03T17:26:10.148Z 23 460-479 74 74 74 135 139 139 T T T <---
81162024-02-03T17:26:10.148Z 24 480-499 74 74 74 135 135 135 F F F
81172024-02-03T17:26:10.148Z 25 500-519 70 70 70 130 130 130 F F F
81182024-02-03T17:26:10.148Z 26 520-539 72 72 72 133 133 133 F F F
81192024-02-03T17:26:10.148Z 27 540-559 70 70 70 128 128 128 F F F
81202024-02-03T17:26:10.148Z 28 560-579 65 65 65 114 114 114 F F F
81212024-02-03T17:26:10.148Z 29 580-599 72 74 74 132 138 138 F F F <---
81222024-02-03T17:26:10.149ZMax gen: 74, Max flush: 139
81232024-02-03T17:26:10.149ZA Difference in extent metadata is expected here
81242024-02-03T17:26:10.149ZOn loop 37
81252024-02-03T17:26:10.149Z
81262024-02-03T17:26:10.149ZVerifying data now
81272024-02-03T17:26:10.149ZNo change after verify
81282024-02-03T17:26:10.149ZLoop: 37 Downstairs dump after verify (and repair):
81292024-02-03T17:26:10.149ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81302024-02-03T17:26:10.149Z 0 000-019 69 69 69 125 125 125 F F F
81312024-02-03T17:26:10.149Z 1 020-039 75 75 75 140 140 140 F F F
81322024-02-03T17:26:10.149Z 2 040-059 63 63 63 110 110 110 F F F
81332024-02-03T17:26:10.149Z 3 060-079 74 74 74 138 138 138 F F F
81342024-02-03T17:26:10.149Z 4 080-099 73 73 73 134 134 134 F F F
81352024-02-03T17:26:10.149Z 5 100-119 73 73 73 134 134 134 F F F
81362024-02-03T17:26:10.149Z 6 120-139 74 74 74 137 137 137 F F F
81372024-02-03T17:26:10.149Z 7 140-159 72 72 72 133 133 133 F F F
81382024-02-03T17:26:10.149Z 8 160-179 75 75 75 140 140 140 F F F
81392024-02-03T17:26:10.149Z 9 180-199 75 75 75 140 140 140 F F F
81402024-02-03T17:26:10.149Z 10 200-219 75 75 75 140 140 140 F F F
81412024-02-03T17:26:10.149Z 11 220-239 70 70 70 129 129 129 F F F
81422024-02-03T17:26:10.149Z 12 240-259 59 59 59 105 105 105 F F F
81432024-02-03T17:26:10.149Z 13 260-279 72 72 72 133 133 133 F F F
81442024-02-03T17:26:10.149Z 14 280-299 72 72 72 133 133 133 F F F
81452024-02-03T17:26:10.150Z 15 300-319 74 74 74 136 136 136 F F F
81462024-02-03T17:26:10.150Z 16 320-339 74 74 74 136 136 136 F F F
81472024-02-03T17:26:10.150Z 17 340-359 73 73 73 134 134 134 F F F
81482024-02-03T17:26:10.150Z 18 360-379 73 73 73 134 134 134 F F F
81492024-02-03T17:26:10.150Z 19 380-399 66 66 66 118 118 118 F F F
81502024-02-03T17:26:10.150Z 20 400-419 75 75 75 140 140 140 F F F
81512024-02-03T17:26:10.150Z 21 420-439 75 75 75 140 140 140 F F F
81522024-02-03T17:26:10.150Z 22 440-459 75 75 75 140 140 140 F F F
81532024-02-03T17:26:10.150Z 23 460-479 75 75 75 140 140 140 F F F
81542024-02-03T17:26:10.150Z 24 480-499 74 74 74 135 135 135 F F F
81552024-02-03T17:26:10.150Z 25 500-519 70 70 70 130 130 130 F F F
81562024-02-03T17:26:10.150Z 26 520-539 72 72 72 133 133 133 F F F
81572024-02-03T17:26:10.150Z 27 540-559 70 70 70 128 128 128 F F F
81582024-02-03T17:26:10.150Z 28 560-579 65 65 65 114 114 114 F F F
81592024-02-03T17:26:10.150Z 29 580-599 74 74 74 138 138 138 F F F
81602024-02-03T17:26:10.150ZMax gen: 75, Max flush: 140
81612024-02-03T17:26:10.150Z
81622024-02-03T17:26:10.150ZBegin loop 38
81632024-02-03T17:26:10.150ZDownstairs to restart: 2
81642024-02-03T17:26:10.150Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.281099123Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436}
81652024-02-03T17:26:10.150Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.281643737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436}
81662024-02-03T17:26:10.150Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.281672506Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436}
81672024-02-03T17:26:10.150Z{"msg":"Crucible d1c2bed5-2d80-47cf-b46c-f2e76714daa9 has session id: 5ae2ef80-1179-4d54-89ae-5d015c9564a1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.281693559Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81682024-02-03T17:26:10.150Z{"msg":"Upstairs opts: Upstairs UUID: d1c2bed5-2d80-47cf-b46c-f2e76714daa9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.281748174Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81692024-02-03T17:26:10.151Z{"msg":"Crucible stats registered with UUID: d1c2bed5-2d80-47cf-b46c-f2e76714daa9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.281783905Z"Crucible runtime is spawned
81702024-02-03T17:26:10.151Z,"hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81712024-02-03T17:26:10.151Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.28196964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436}
81722024-02-03T17:26:10.151Z{"msg":"d1c2bed5-2d80-47cf-b46c-f2e76714daa9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282057226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81732024-02-03T17:26:10.151Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.28214156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81742024-02-03T17:26:10.151Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282184478Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81752024-02-03T17:26:10.151Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.28222311Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81762024-02-03T17:26:10.151Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282267245Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81772024-02-03T17:26:10.151Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282308681Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81782024-02-03T17:26:10.151Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282352582Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81792024-02-03T17:26:10.151Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282395578Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81802024-02-03T17:26:10.151Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.28243748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81812024-02-03T17:26:10.151Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282476828Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81822024-02-03T17:26:10.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282721183Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"io task","client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81832024-02-03T17:26:10.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282774005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"io task","client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81842024-02-03T17:26:10.151Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.282815423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"io task","client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81852024-02-03T17:26:10.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.284160005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81862024-02-03T17:26:10.151Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.284198101Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81872024-02-03T17:26:10.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.284221531Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81882024-02-03T17:26:10.151Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.284255042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81892024-02-03T17:26:10.151Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.284686867Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81902024-02-03T17:26:10.151Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.284723256Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81912024-02-03T17:26:10.151Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.285498224Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81922024-02-03T17:26:10.151Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.285550663Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81932024-02-03T17:26:10.152Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.285957723Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81942024-02-03T17:26:10.152Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.28672834Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81952024-02-03T17:26:10.152Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.286775516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81962024-02-03T17:26:10.152Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:05.28680252Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81972024-02-03T17:26:10.152ZWait for a query_work_queue command to finish before sending IO
81982024-02-03T17:26:10.152Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156424277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
81992024-02-03T17:26:10.152Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156514843Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82002024-02-03T17:26:10.152Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156540343Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82012024-02-03T17:26:10.152Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156561143Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82022024-02-03T17:26:10.163Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156583485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82032024-02-03T17:26:10.163Z{"msg":"[0]R flush_numbers[0..12]: [125, 140, 110, 138, 134, 134, 137, 133, 140, 140, 140, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156607478Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82042024-02-03T17:26:10.163Z{"msg":"[0]R generation[0..12]: [69, 75, 63, 74, 73, 73, 74, 72, 75, 75, 75, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156630814Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82052024-02-03T17:26:10.163Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.15665468Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82062024-02-03T17:26:10.163Z{"msg":"[1]R flush_numbers[0..12]: [125, 140, 110, 138, 134, 134, 137, 133, 140, 140, 140, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156678963Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82072024-02-03T17:26:10.163Z{"msg":"[1]R generation[0..12]: [69, 75, 63, 74, 73, 73, 74, 72, 75, 75, 75, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156701874Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82082024-02-03T17:26:10.163Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156725877Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82092024-02-03T17:26:10.163Z{"msg":"[2]R flush_numbers[0..12]: [125, 140, 110, 138, 134, 134, 137, 133, 140, 140, 140, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156750496Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82102024-02-03T17:26:10.163Z{"msg":"[2]R generation[0..12]: [69, 75, 63, 74, 73, 73, 74, 72, 75, 75, 75, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156786667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82112024-02-03T17:26:10.163Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156812453Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82122024-02-03T17:26:10.164Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156834414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82132024-02-03T17:26:10.164Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156856696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82142024-02-03T17:26:10.164Z{"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156880366Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82152024-02-03T17:26:10.164Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156918417Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"":"downstairs","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82162024-02-03T17:26:10.164Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156951868Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82172024-02-03T17:26:10.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.156985209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"0","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82182024-02-03T17:26:10.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.157023879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"1","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82192024-02-03T17:26:10.164Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.157068287Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"client":"2","":"downstairs","session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82202024-02-03T17:26:10.164Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.15711174Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82212024-02-03T17:26:10.164Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82222024-02-03T17:26:10.164Z,"time":"2024-02-03T17:26:09.157143762Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82232024-02-03T17:26:10.164Z{"msg":"d1c2bed5-2d80-47cf-b46c-f2e76714daa9 is now active with session: 5ae2ef80-1179-4d54-89ae-5d015c9564a1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.157179157Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82242024-02-03T17:26:10.164Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.157213321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436,"session_id":"5ae2ef80-1179-4d54-89ae-5d015c9564a1"}
82252024-02-03T17:26:10.164Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:09.157236615Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436}
82262024-02-03T17:26:10.164ZLoading write count information from file "/tmp/test_repair_verify.data"
82272024-02-03T17:26:10.164ZRun Repair workload
82282024-02-03T17:26:10.164Z01/30 Write block 37 len 16384 data: 6 7 8 7
82292024-02-03T17:26:10.164Z02/30 Write block 430 len 16384 data: 4 4 4 3
82302024-02-03T17:26:10.164Z03/30 Read block 222 len 16384
82312024-02-03T17:26:10.165Z04/30 Write block 78 len 28672 data: 6 5 5 5 6 8 9
82322024-02-03T17:26:10.165Z05/30 Flush
82332024-02-03T17:26:10.165Z06/30 Flush
82342024-02-03T17:26:10.165Z07/30 Write block 94 len 20480 data: 8 7 6 5 4
82352024-02-03T17:26:10.165Z08/30 Read block 463 len 32768
82362024-02-03T17:26:10.165Z09/30 Write block 10 len 20480 data: 10 9 8 8 6
82372024-02-03T17:26:10.165Z10/30 Read block 251 len 12288
82382024-02-03T17:26:10.165Z11/30 Read block 122 len 28672
82392024-02-03T17:26:10.165Z12/30 Write block 391 len 16384 data: 4 5 6 7
82402024-02-03T17:26:10.165Z13/30 Read block 75 len 8192
82412024-02-03T17:26:10.165Z14/30 Read block 34 len 16384
82422024-02-03T17:26:10.165Z15/30 Write block 216 len 40960 data: 5 6 6 7 7 11 11 8 9 10
82432024-02-03T17:26:10.165Z16/30 Write block 566 len 16384 data: 7 7 6 5
82442024-02-03T17:26:10.165Z17/30 Read block 16 len 28672
82452024-02-03T17:26:10.165Z18/30 Write block 515 len 12288 data: 4 2 3
82462024-02-03T17:26:10.165Z19/30 Write block 268 len 24576 data: 7 9 12 13 13 13
82472024-02-03T17:26:10.165Z20/30 Write block 279 len 20480 data: 9 7 9 10 9
82482024-02-03T17:26:10.165Z21/30 Write block 395 len 4096 data: 9
82492024-02-03T17:26:10.165Z22/30 Read block 298 len 36864
82502024-02-03T17:26:10.165Z23/30 Write block 161 len 12288 data: 8 9 7
82512024-02-03T17:26:10.166Z24/30 Read block 107 len 20480
82522024-02-03T17:26:10.166Z25/30 Read block 337 len 24576
82532024-02-03T17:26:10.166Z26/30 Write block 555 len 40960 data: 8 8 7 7 8 5 3 2 5 5
82542024-02-03T17:26:10.166Z27/30 Read block 248 len 20480
82552024-02-03T17:26:10.166Z28/30 Read block 22 len 24576
82562024-02-03T17:26:10.166Z29/30 Write block 542 len 36864 data: 6 5 6 6 5 5 6 6 6
82572024-02-03T17:26:10.166Z30/30 Write block 105 len 24576 data: 9 6 8 9 9 9
82582024-02-03T17:26:10.166Z----------------------------------------------------------------
82592024-02-03T17:26:10.166Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:31
82602024-02-03T17:26:10.166ZStates: Active Active Active
82612024-02-03T17:26:10.166ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82622024-02-03T17:26:10.166Z 1 Acked 1000 Write 4 Done Done Done false
82632024-02-03T17:26:10.166Z 2 Acked 1001 Write 4 Done Done Done false
82642024-02-03T17:26:10.166Z 3 Acked 1002 Read 4 Done Done Done false
82652024-02-03T17:26:10.166Z 4 Acked 1003 Write 7 Done Done Sent false
82662024-02-03T17:26:10.166Z 5 Acked 1004 Flush 0 Done Done Sent false
82672024-02-03T17:26:10.166Z 6 Acked 1005 Flush 0 Done Done Sent false
82682024-02-03T17:26:10.166Z 7 Acked 1006 Write 5 Done Done Sent false
82692024-02-03T17:26:10.166Z 8 Acked 1007 Read 8 Done Done Sent false
82702024-02-03T17:26:10.166Z 9 Acked 1008 Write 5 Done Done Sent false
82712024-02-03T17:26:10.166Z 10 Acked 1009 Read 3 Done Done Sent false
82722024-02-03T17:26:10.166Z 11 Acked 1010 Read 7 Done Done Sent false
82732024-02-03T17:26:10.166Z 12 Acked 1011 Write 4 Done Done Sent false
82742024-02-03T17:26:10.166Z 13 Acked 1012 Read 2 Done Done Sent false
82752024-02-03T17:26:10.166Z 14 Acked 1013 Read 4 Done Done Sent false
82762024-02-03T17:26:10.167Z 15 Acked 1014 Write 10 Done Done Sent false
82772024-02-03T17:26:10.167Z 16 Acked 1015 Write 4 Done Done Sent false
82782024-02-03T17:26:10.167Z 17 Acked 1016 Read 7 Done Done Sent false
82792024-02-03T17:26:10.167Z 18 Acked 1017 Write 3 Done Done Sent false
82802024-02-03T17:26:10.167Z 19 Acked 1018 Write 6 Done Done Sent false
82812024-02-03T17:26:10.167Z 20 Acked 1019 Write 5 Done Done Sent false
82822024-02-03T17:26:10.167Z 21 Acked 1020 Write 1 Done Done Sent false
82832024-02-03T17:26:10.167Z 22 Acked 1021 Read 9 Done Done Sent false
82842024-02-03T17:26:10.167Z 23 Acked 1022 Write 3 Done Done Sent false
82852024-02-03T17:26:10.167Z 24 Acked 1023 Read 5 Done Done Sent false
82862024-02-03T17:26:10.167Z 25 Acked 1024 Read 6 Done Done Sent false
82872024-02-03T17:26:10.167Z 26 Acked 1025 Write 10 Done Done Sent false
82882024-02-03T17:26:10.167Z 27 Acked 1026 Read 5 Done Done Sent false
82892024-02-03T17:26:10.167Z 28 Acked 1027 Flush 0 Done Done Sent false
82902024-02-03T17:26:10.167Z 29 Acked 1028 Read 6 Done Done Sent false
82912024-02-03T17:26:10.167Z 30 Acked 1029 Write 9 Sent Done Sent false
82922024-02-03T17:26:10.167Z 31 Acked 1030 Write 6 Sent Sent Sent false
82932024-02-03T17:26:10.167Z STATES DS:0 DS:1 DS:2 TOTAL
82942024-02-03T17:26:10.167Z New 0 0 0 0
82952024-02-03T17:26:10.167Z Sent 2 1 28 31
82962024-02-03T17:26:10.167Z Done 29 30 3 62
82972024-02-03T17:26:10.167Z Skipped 0 0 0 0
82982024-02-03T17:26:10.167Z Error 0 0 0 0
82992024-02-03T17:26:10.167ZLast Flush: 1027 1027 0
83002024-02-03T17:26:10.168ZDownstairs last five completed:
83012024-02-03T17:26:10.168ZUpstairs last five completed: 31 30 29 28 27
83022024-02-03T17:26:10.182Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:09.300476255Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1436}
83032024-02-03T17:26:10.182ZWrote out file "/tmp/test_repair_verify.data"
83042024-02-03T17:26:10.182Z
83052024-02-03T17:26:10.182ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83062024-02-03T17:26:10.182ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83072024-02-03T17:26:10.182Z 0 000-019 76 76 69 143 143 125 F F F <---
83082024-02-03T17:26:10.182Z 1 020-039 76 76 75 141 141 140 F F T <---
83092024-02-03T17:26:10.182Z 2 040-059 76 76 63 141 141 110 F F T <---
83102024-02-03T17:26:10.182Z 3 060-079 76 76 74 141 141 138 F F F <---
83112024-02-03T17:26:10.183Z 4 080-099 76 76 73 143 143 134 F F F <---
83122024-02-03T17:26:10.183Z 5 100-119 73 73 73 134 134 134 T T F <---
83132024-02-03T17:26:10.183Z 6 120-139 74 74 74 137 137 137 F F F
83142024-02-03T17:26:10.183Z 7 140-159 72 72 72 133 133 133 F F F
83152024-02-03T17:26:10.183Z 8 160-179 76 76 75 143 143 140 F F F <---
83162024-02-03T17:26:10.183Z 9 180-199 75 75 75 140 140 140 F F F
83172024-02-03T17:26:10.183Z 10 200-219 76 76 75 143 143 140 F F F <---
83182024-02-03T17:26:10.183Z 11 220-239 76 76 70 143 143 129 F F F <---
83192024-02-03T17:26:10.183Z 12 240-259 59 59 59 105 105 105 F F F
83202024-02-03T17:26:10.183Z 13 260-279 76 76 72 143 143 133 F F F <---
83212024-02-03T17:26:10.183Z 14 280-299 76 76 72 143 143 133 F F F <---
83222024-02-03T17:26:10.183Z 15 300-319 74 74 74 136 136 136 F F F
83232024-02-03T17:26:10.183Z 16 320-339 74 74 74 136 136 136 F F F
83242024-02-03T17:26:10.183Z 17 340-359 73 73 73 134 134 134 F F F
83252024-02-03T17:26:10.183Z 18 360-379 73 73 73 134 134 134 F F F
83262024-02-03T17:26:10.183Z 19 380-399 76 76 66 143 143 118 F F F <---
83272024-02-03T17:26:10.183Z 20 400-419 75 75 75 140 140 140 F F F
83282024-02-03T17:26:10.183Z 21 420-439 76 76 75 141 141 140 F F T <---
83292024-02-03T17:26:10.183Z 22 440-459 75 75 75 140 140 140 F F F
83302024-02-03T17:26:10.183Z 23 460-479 75 75 75 140 140 140 F F F
83312024-02-03T17:26:10.183Z 24 480-499 74 74 74 135 135 135 F F F
83322024-02-03T17:26:10.183Z 25 500-519 76 76 70 143 143 130 F F F <---
83332024-02-03T17:26:10.184Z 26 520-539 72 72 72 133 133 133 F F F
83342024-02-03T17:26:10.184Z 27 540-559 76 76 70 143 143 128 T T F <---
83352024-02-03T17:26:10.184Z 28 560-579 76 76 65 143 143 114 F F F <---
83362024-02-03T17:26:10.184Z 29 580-599 74 74 74 138 138 138 F F F
83372024-02-03T17:26:10.184ZMax gen: 76, Max flush: 143
83382024-02-03T17:26:10.184ZError: Difference in extent metadata found!
83392024-02-03T17:26:10.194ZA Difference in extent metadata is expected here
83402024-02-03T17:26:10.194ZOn loop 38
83412024-02-03T17:26:10.194Z
83422024-02-03T17:26:10.194ZVerifying data now
83432024-02-03T17:26:16.553ZNo change after verify
83442024-02-03T17:26:16.553ZLoop: 38 Downstairs dump after verify (and repair):
83452024-02-03T17:26:16.577ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83462024-02-03T17:26:16.577Z 0 000-019 76 76 76 143 143 143 F F F
83472024-02-03T17:26:16.578Z 1 020-039 76 76 76 141 141 141 F F F
83482024-02-03T17:26:16.578Z 2 040-059 76 76 76 141 141 141 F F F
83492024-02-03T17:26:16.582Z 3 060-079 76 76 76 141 141 141 F F F
83502024-02-03T17:26:16.583Z 4 080-099 76 76 76 143 143 143 F F F
83512024-02-03T17:26:16.583Z 5 100-119 77 77 77 144 144 144 F F F
83522024-02-03T17:26:16.583Z 6 120-139 74 74 74 137 137 137 F F F
83532024-02-03T17:26:16.583Z 7 140-159 72 72 72 133 133 133 F F F
83542024-02-03T17:26:16.583Z 8 160-179 76 76 76 143 143 143 F F F
83552024-02-03T17:26:16.583Z 9 180-199 75 75 75 140 140 140 F F F
83562024-02-03T17:26:16.583Z 10 200-219 76 76 76 143 143 143 F F F
83572024-02-03T17:26:16.583Z 11 220-239 76 76 76 143 143 143 F F F
83582024-02-03T17:26:16.583Z 12 240-259 59 59 59 105 105 105 F F F
83592024-02-03T17:26:16.583Z 13 260-279 76 76 76 143 143 143 F F F
83602024-02-03T17:26:16.583Z 14 280-299 76 76 76 143 143 143 F F F
83612024-02-03T17:26:16.583Z 15 300-319 74 74 74 136 136 136 F F F
83622024-02-03T17:26:16.583Z 16 320-339 74 74 74 136 136 136 F F F
83632024-02-03T17:26:16.583Z 17 340-359 73 73 73 134 134 134 F F F
83642024-02-03T17:26:16.583Z 18 360-379 73 73 73 134 134 134 F F F
83652024-02-03T17:26:16.583Z 19 380-399 76 76 76 143 143 143 F F F
83662024-02-03T17:26:16.583Z 20 400-419 75 75 75 140 140 140 F F F
83672024-02-03T17:26:16.583Z 21 420-439 76 76 76 141 141 141 F F F
83682024-02-03T17:26:16.583Z 22 440-459 75 75 75 140 140 140 F F F
83692024-02-03T17:26:16.583Z 23 460-479 75 75 75 140 140 140 F F F
83702024-02-03T17:26:16.583Z 24 480-499 74 74 74 135 135 135 F F F
83712024-02-03T17:26:16.583Z 25 500-519 76 76 76 143 143 143 F F F
83722024-02-03T17:26:16.583Z 26 520-539 72 72 72 133 133 133 F F F
83732024-02-03T17:26:16.584Z 27 540-559 77 77 77 144 144 144 F F F
83742024-02-03T17:26:16.584Z 28 560-579 76 76 76 143 143 143 F F F
83752024-02-03T17:26:16.584Z 29 580-599 74 74 74 138 138 138 F F F
83762024-02-03T17:26:16.584ZMax gen: 77, Max flush: 144
83772024-02-03T17:26:16.587Z
83782024-02-03T17:26:16.587ZBegin loop 39
83792024-02-03T17:26:16.587ZDownstairs to restart: 1
83802024-02-03T17:26:17.026Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.783494745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444}
83812024-02-03T17:26:17.026Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784016489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444}
83822024-02-03T17:26:17.033Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784045622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444}
83832024-02-03T17:26:17.033Z{"msg":"Crucible 06043433-3de5-4f67-ba54-661bb9196356 has session id: 42ebf89b-6356-481a-a02e-e5214811d795","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784112837Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444Crucible runtime is spawned
83842024-02-03T17:26:17.033Z,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83852024-02-03T17:26:17.033Z{"msg":"Upstairs opts: Upstairs UUID: 06043433-3de5-4f67-ba54-661bb9196356, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784271461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83862024-02-03T17:26:17.033Z{"msg":"Crucible stats registered with UUID: 06043433-3de5-4f67-ba54-661bb9196356","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784306946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83872024-02-03T17:26:17.033Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784404315Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444}
83882024-02-03T17:26:17.033Z{"msg":"06043433-3de5-4f67-ba54-661bb9196356 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784547654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83892024-02-03T17:26:17.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784609697Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83902024-02-03T17:26:17.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784636651Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83912024-02-03T17:26:17.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784658771Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83922024-02-03T17:26:17.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784715747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83932024-02-03T17:26:17.033Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.78476125Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83942024-02-03T17:26:17.033Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784803858Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83952024-02-03T17:26:17.033Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.78484889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83962024-02-03T17:26:17.033Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784914632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83972024-02-03T17:26:17.033Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.784956929Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83982024-02-03T17:26:17.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.785293814Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"io task","client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
83992024-02-03T17:26:17.033Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.785582521Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"io task","client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84002024-02-03T17:26:17.034Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.785693536Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"io task","client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84012024-02-03T17:26:17.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.786291691Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84022024-02-03T17:26:17.034Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.786376311Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84032024-02-03T17:26:17.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.786450023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84042024-02-03T17:26:17.034Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.786528624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84052024-02-03T17:26:17.034Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.787554954Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84062024-02-03T17:26:17.034Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.787659184Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84072024-02-03T17:26:17.034Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.788320637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84082024-02-03T17:26:17.034Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.78838776Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84092024-02-03T17:26:17.034Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.788648063Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84102024-02-03T17:26:17.034Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.788726528Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84112024-02-03T17:26:17.034Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.7887652Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84122024-02-03T17:26:17.034Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.788810846Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84132024-02-03T17:26:17.034Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.788835999Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84142024-02-03T17:26:17.034Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.78889233Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84152024-02-03T17:26:17.034Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.789278423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84162024-02-03T17:26:17.034Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790054796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84172024-02-03T17:26:17.034Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790117633Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84182024-02-03T17:26:17.034Z{"msg":"[0]R flush_numbers[0..12]: [143, 141, 141, 141, 143, 144, 137, 133, 143, 140, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790145383Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84192024-02-03T17:26:17.034Z{"msg":"[0]R generation[0..12]: [76, 76, 76, 76, 76, 77, 74, 72, 76, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790187955Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84202024-02-03T17:26:17.034Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790214014Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84212024-02-03T17:26:17.034Z{"msg":"[1]R flush_numbers[0..12]: [143, 141, 141, 141, 143, 144, 137, 133, 143, 140, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.79023822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84222024-02-03T17:26:17.034Z{"msg":"[1]R generation[0..12]: [76, 76, 76, 76, 76, 77, 74, 72, 76, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790281137Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84232024-02-03T17:26:17.035Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790307063Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84242024-02-03T17:26:17.035Z{"msg":"[2]R flush_numbers[0..12]: [143, 141, 141, 141, 143, 144, 137, 133, 143, 140, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790330868Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84252024-02-03T17:26:17.035Z{"msg":"[2]R generation[0..12]: [76, 76, 76, 76, 76, 77, 74, 72, 76, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790379377Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84262024-02-03T17:26:17.035Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
84272024-02-03T17:26:17.035Z,"time":"2024-02-03T17:26:15.790404845Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84282024-02-03T17:26:17.035Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790430958Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84292024-02-03T17:26:17.035Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790453268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84302024-02-03T17:26:17.035Z{"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790475746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84312024-02-03T17:26:17.035Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790497485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"":"downstairs","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84322024-02-03T17:26:17.035Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790519544Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84332024-02-03T17:26:17.035Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790560008Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"0","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84342024-02-03T17:26:17.035Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790584585Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"1","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84352024-02-03T17:26:17.035Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790606794Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"client":"2","":"downstairs","session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84362024-02-03T17:26:17.035Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790628549Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84372024-02-03T17:26:17.035Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790648829Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84382024-02-03T17:26:17.035Z{"msg":"06043433-3de5-4f67-ba54-661bb9196356 is now active with session: 42ebf89b-6356-481a-a02e-e5214811d795","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790671721Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84392024-02-03T17:26:17.035Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790691326Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444,"session_id":"42ebf89b-6356-481a-a02e-e5214811d795"}
84402024-02-03T17:26:17.035Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:15.790711513Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444}
84412024-02-03T17:26:17.035ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84422024-02-03T17:26:17.035ZLoading write count information from file "/tmp/test_repair_verify.data"
84432024-02-03T17:26:17.035ZRun Repair workload
84442024-02-03T17:26:17.035Z01/30 Write block 523 len 28672 data: 5 5 6 8 9 8 9
84452024-02-03T17:26:17.035Z02/30 Write block 366 len 32768 data: 7 7 7 6 5 5 5 4
84462024-02-03T17:26:17.035Z03/30 Write block 32 len 12288 data: 5 4 4
84472024-02-03T17:26:17.035Z04/30 Read block 268 len 12288
84482024-02-03T17:26:17.036Z05/30 Write block 314 len 16384 data: 5 5 5 4
84492024-02-03T17:26:17.036Z06/30 Write block 301 len 36864 data: 6 7 7 6 7 8 5 6 6
84502024-02-03T17:26:17.036Z07/30 Write block 481 len 12288 data: 9 7 6
84512024-02-03T17:26:17.036Z08/30 Read block 201 len 20480
84522024-02-03T17:26:17.036Z09/30 Read block 274 len 24576
84532024-02-03T17:26:17.036Z10/30 Flush
84542024-02-03T17:26:17.036Z11/30 Write block 103 len 16384 data: 6 7 10 7
84552024-02-03T17:26:17.036Z12/30 Read block 506 len 24576
84562024-02-03T17:26:17.036Z13/30 Read block 457 len 8192
84572024-02-03T17:26:17.036Z14/30 Read block 58 len 16384
84582024-02-03T17:26:17.036Z15/30 Read block 445 len 16384
84592024-02-03T17:26:17.036Z16/30 Read block 22 len 24576
84602024-02-03T17:26:17.036Z17/30 Read block 235 len 36864
84612024-02-03T17:26:17.036Z18/30 Read block 480 len 8192
84622024-02-03T17:26:17.036Z19/30 Read block 364 len 40960
84632024-02-03T17:26:17.036Z20/30 Read block 262 len 16384
84642024-02-03T17:26:17.036Z21/30 Flush
84652024-02-03T17:26:17.036Z22/30 Write block 206 len 12288 data: 7 9 9
84662024-02-03T17:26:17.036Z23/30 Read block 304 len 40960
84672024-02-03T17:26:17.036Z24/30 Read block 315 len 4096
84682024-02-03T17:26:17.036Z25/30 Write block 115 len 36864 data: 10 8 9 9 9 8 8 5 6
84692024-02-03T17:26:17.036Z26/30 Flush
84702024-02-03T17:26:17.036Z27/30 Write block 418 len 16384 data: 7 7 7 5
84712024-02-03T17:26:17.036Z28/30 Write block 354 len 40960 data: 6 7 10 10 11 12 11 12 11 13
84722024-02-03T17:26:17.036Z29/30 Write block 10 len 12288 data: 11 10 9
84732024-02-03T17:26:17.037Z30/30 Read block 425 len 32768
84742024-02-03T17:26:17.037Z----------------------------------------------------------------
84752024-02-03T17:26:17.037Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84762024-02-03T17:26:17.037ZStates: Active Active Active
84772024-02-03T17:26:17.037ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84782024-02-03T17:26:17.037Z 1 Acked 1000 Write 7 Done Done Done false
84792024-02-03T17:26:17.037Z 2 Acked 1001 Write 8 Done Sent Done false
84802024-02-03T17:26:17.037Z 3 Acked 1002 Write 3 Done Sent Done false
84812024-02-03T17:26:17.037Z 4 Acked 1003 Read 3 Done Sent Done false
84822024-02-03T17:26:17.047Z 5 Acked 1004 Write 4 Done Sent Done false
84832024-02-03T17:26:17.047Z 6 Acked 1005 Write 9 Done Sent Done false
84842024-02-03T17:26:17.047Z 7 Acked 1006 Write 3 Done Sent Done false
84852024-02-03T17:26:17.047Z 8 Acked 1007 Read 5 Done Sent Done false
84862024-02-03T17:26:17.047Z 9 Acked 1008 Read 6 Done Sent Done false
84872024-02-03T17:26:17.047Z 10 Acked 1009 Flush 0 Done Sent Done false
84882024-02-03T17:26:17.047Z 11 Acked 1010 Write 4 Done Sent Done false
84892024-02-03T17:26:17.048Z 12 Acked 1011 Read 6 Done Sent Done false
84902024-02-03T17:26:17.048Z 13 Acked 1012 Read 2 Done Sent Done false
84912024-02-03T17:26:17.048Z 14 Acked 1013 Read 4 Done Sent Done false
84922024-02-03T17:26:17.048Z 15 Acked 1014 Read 4 Done Sent Done false
84932024-02-03T17:26:17.048Z 16 Acked 1015 Read 6 Done Sent Done false
84942024-02-03T17:26:17.048Z 17 Acked 1016 Read 9 Done Sent Done false
84952024-02-03T17:26:17.048Z 18 Acked 1017 Read 2 Done Sent Done false
84962024-02-03T17:26:17.048Z 19 Acked 1018 Read 10 Done Sent Done false
84972024-02-03T17:26:17.048Z 20 Acked 1019 Read 4 Done Sent Done false
84982024-02-03T17:26:17.048Z 21 Acked 1020 Flush 0 Done Sent Done false
84992024-02-03T17:26:17.048Z 22 Acked 1021 Write 3 Done Sent Done false
85002024-02-03T17:26:17.048Z 23 Acked 1022 Read 10 Done Sent Done false
85012024-02-03T17:26:17.049Z 24 Acked 1023 Read 1 Done Sent Done false
85022024-02-03T17:26:17.049Z 25 Acked 1024 Write 9 Done Sent Done false
85032024-02-03T17:26:17.049Z 26 Acked 1025 Flush 0 Done Sent Done false
85042024-02-03T17:26:17.049Z 27 Acked 1026 Write 4 Done Sent Done false
85052024-02-03T17:26:17.049Z 28 Acked 1027 Write 10 Done Sent Done false
85062024-02-03T17:26:17.049Z 29 Acked 1028 Write 3 Done Sent Done false
85072024-02-03T17:26:17.049Z 30 Acked 1029 Read 8 Done Sent Done false
85082024-02-03T17:26:17.049Z STATES DS:0 DS:1 DS:2 TOTAL
85092024-02-03T17:26:17.049Z New 0 0 0 0
85102024-02-03T17:26:17.050Z Sent 0 29 0 29
85112024-02-03T17:26:17.050Z Done 30 1 30 61
85122024-02-03T17:26:17.050Z Skipped 0 0 0 0
85132024-02-03T17:26:17.050Z Error 0 0 0 0
85142024-02-03T17:26:17.050ZLast Flush: 1025 0 1025
85152024-02-03T17:26:17.050ZDownstairs last five completed:
85162024-02-03T17:26:17.050ZUpstairs last five completed: 30 29 28 27 26
85172024-02-03T17:26:17.050Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:15.839786304Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1444}
85182024-02-03T17:26:17.050ZWrote out file "/tmp/test_repair_verify.data"
85192024-02-03T17:26:17.050Z
85202024-02-03T17:26:17.050ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85212024-02-03T17:26:17.050ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85222024-02-03T17:26:17.051Z 0 000-019 76 76 76 143 143 143 T F T <---
85232024-02-03T17:26:17.051Z 1 020-039 78 76 78 145 141 145 F F F <---
85242024-02-03T17:26:17.051Z 2 040-059 76 76 76 141 141 141 F F F
85252024-02-03T17:26:17.051Z 3 060-079 76 76 76 141 141 141 F F F
85262024-02-03T17:26:17.051Z 4 080-099 76 76 76 143 143 143 F F F
85272024-02-03T17:26:17.051Z 5 100-119 78 77 78 147 144 147 F F F <---
85282024-02-03T17:26:17.051Z 6 120-139 78 74 78 147 137 147 F F F <---
85292024-02-03T17:26:17.051Z 7 140-159 72 72 72 133 133 133 F F F
85302024-02-03T17:26:17.051Z 8 160-179 76 76 76 143 143 143 F F F
85312024-02-03T17:26:17.051Z 9 180-199 75 75 75 140 140 140 F F F
85322024-02-03T17:26:17.051Z 10 200-219 78 76 78 147 143 147 F F F <---
85332024-02-03T17:26:17.051Z 11 220-239 76 76 76 143 143 143 F F F
85342024-02-03T17:26:17.051Z 12 240-259 59 59 59 105 105 105 F F F
85352024-02-03T17:26:17.052Z 13 260-279 76 76 76 143 143 143 F F F
85362024-02-03T17:26:17.052Z 14 280-299 76 76 76 143 143 143 F F F
85372024-02-03T17:26:17.052Z 15 300-319 78 74 78 145 136 145 F F F <---
85382024-02-03T17:26:17.052Z 16 320-339 74 74 74 136 136 136 F F F
85392024-02-03T17:26:17.052Z 17 340-359 73 73 73 134 134 134 T F T <---
85402024-02-03T17:26:17.052Z 18 360-379 78 73 78 145 134 145 T F T <---
85412024-02-03T17:26:17.052Z 19 380-399 76 76 76 143 143 143 F F F
85422024-02-03T17:26:17.052Z 20 400-419 75 75 75 140 140 140 T F T <---
85432024-02-03T17:26:17.052Z 21 420-439 76 76 76 141 141 141 T F T <---
85442024-02-03T17:26:17.053Z 22 440-459 75 75 75 140 140 140 F F F
85452024-02-03T17:26:17.053Z 23 460-479 75 75 75 140 140 140 F F F
85462024-02-03T17:26:17.053Z 24 480-499 78 74 78 145 135 145 F F F <---
85472024-02-03T17:26:17.053Z 25 500-519 76 76 76 143 143 143 F F F
85482024-02-03T17:26:17.053Z 26 520-539 78 72 78 145 133 145 F T F <---
85492024-02-03T17:26:17.053Z 27 540-559 77 77 77 144 144 144 F F F
85502024-02-03T17:26:17.053Z 28 560-579 76 76 76 143 143 143 F F F
85512024-02-03T17:26:17.053Z 29 580-599 74 74 74 138 138 138 F F F
85522024-02-03T17:26:17.053ZMax gen: 78, Max flush: 147
85532024-02-03T17:26:17.054ZError: Difference in extent metadata found!
85542024-02-03T17:26:17.054ZA Difference in extent metadata is expected here
85552024-02-03T17:26:17.054ZOn loop 39
85562024-02-03T17:26:17.054Z
85572024-02-03T17:26:17.054ZVerifying data now
85582024-02-03T17:26:21.802ZNo change after verify
85592024-02-03T17:26:21.802ZLoop: 39 Downstairs dump after verify (and repair):
85602024-02-03T17:26:21.826ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85612024-02-03T17:26:21.826Z 0 000-019 79 79 79 148 148 148 F F F
85622024-02-03T17:26:21.826Z 1 020-039 78 78 78 145 145 145 F F F
85632024-02-03T17:26:21.826Z 2 040-059 76 76 76 141 141 141 F F F
85642024-02-03T17:26:21.826Z 3 060-079 76 76 76 141 141 141 F F F
85652024-02-03T17:26:21.830Z 4 080-099 76 76 76 143 143 143 F F F
85662024-02-03T17:26:21.830Z 5 100-119 78 78 78 147 147 147 F F F
85672024-02-03T17:26:21.830Z 6 120-139 78 78 78 147 147 147 F F F
85682024-02-03T17:26:21.830Z 7 140-159 72 72 72 133 133 133 F F F
85692024-02-03T17:26:21.830Z 8 160-179 76 76 76 143 143 143 F F F
85702024-02-03T17:26:21.830Z 9 180-199 75 75 75 140 140 140 F F F
85712024-02-03T17:26:21.830Z 10 200-219 78 78 78 147 147 147 F F F
85722024-02-03T17:26:21.830Z 11 220-239 76 76 76 143 143 143 F F F
85732024-02-03T17:26:21.830Z 12 240-259 59 59 59 105 105 105 F F F
85742024-02-03T17:26:21.830Z 13 260-279 76 76 76 143 143 143 F F F
85752024-02-03T17:26:21.830Z 14 280-299 76 76 76 143 143 143 F F F
85762024-02-03T17:26:21.830Z 15 300-319 78 78 78 145 145 145 F F F
85772024-02-03T17:26:21.830Z 16 320-339 74 74 74 136 136 136 F F F
85782024-02-03T17:26:21.830Z 17 340-359 79 79 79 148 148 148 F F F
85792024-02-03T17:26:21.830Z 18 360-379 79 79 79 148 148 148 F F F
85802024-02-03T17:26:21.831Z 19 380-399 76 76 76 143 143 143 F F F
85812024-02-03T17:26:21.831Z 20 400-419 79 79 79 148 148 148 F F F
85822024-02-03T17:26:21.831Z 21 420-439 79 79 79 148 148 148 F F F
85832024-02-03T17:26:21.831Z 22 440-459 75 75 75 140 140 140 F F F
85842024-02-03T17:26:21.831Z 23 460-479 75 75 75 140 140 140 F F F
85852024-02-03T17:26:21.831Z 24 480-499 78 78 78 145 145 145 F F F
85862024-02-03T17:26:21.831Z 25 500-519 76 76 76 143 143 143 F F F
85872024-02-03T17:26:21.831Z 26 520-539 78 78 78 145 145 145 F F F
85882024-02-03T17:26:21.831Z 27 540-559 77 77 77 144 144 144 F F F
85892024-02-03T17:26:21.831Z 28 560-579 76 76 76 143 143 143 F F F
85902024-02-03T17:26:21.831Z 29 580-599 74 74 74 138 138 138 F F F
85912024-02-03T17:26:21.831ZMax gen: 79, Max flush: 148
85922024-02-03T17:26:21.836Z
85932024-02-03T17:26:21.836ZBegin loop 40
85942024-02-03T17:26:21.836ZDownstairs to restart: 0
85952024-02-03T17:26:21.865Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.029248006Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452}
85962024-02-03T17:26:21.865Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.02987286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452}
85972024-02-03T17:26:21.865Z{Crucible runtime is spawned
85982024-02-03T17:26:21.870Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.029990939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452}
85992024-02-03T17:26:21.870Z{"msg":"Crucible 36102da0-d798-47ff-9f76-56e40305fd22 has session id: 7f760491-2679-42d4-b4e3-ba2f02109f9f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030058247Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86002024-02-03T17:26:21.870Z{"msg":"Upstairs opts: Upstairs UUID: 36102da0-d798-47ff-9f76-56e40305fd22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030118811Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86012024-02-03T17:26:21.870Z{"msg":"Crucible stats registered with UUID: 36102da0-d798-47ff-9f76-56e40305fd22","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030190811Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86022024-02-03T17:26:21.870Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030244981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452}
86032024-02-03T17:26:21.870Z{"msg":"36102da0-d798-47ff-9f76-56e40305fd22 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030343552Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86042024-02-03T17:26:21.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030393533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86052024-02-03T17:26:21.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030420042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86062024-02-03T17:26:21.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030469621Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86072024-02-03T17:26:21.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030494224Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86082024-02-03T17:26:21.871Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030529595Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86092024-02-03T17:26:21.871Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030566022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86102024-02-03T17:26:21.871Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030589535Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86112024-02-03T17:26:21.871Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030613964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86122024-02-03T17:26:21.871Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030666595Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86132024-02-03T17:26:21.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.03085964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"io task","client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86142024-02-03T17:26:21.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.030983489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"io task","client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86152024-02-03T17:26:21.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.031097444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"io task","client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86162024-02-03T17:26:21.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.032313201Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86172024-02-03T17:26:21.871Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.032520162Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86182024-02-03T17:26:21.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.032594728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86192024-02-03T17:26:21.872Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.032677619Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86202024-02-03T17:26:21.872Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.033378297Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86212024-02-03T17:26:21.872Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.033452753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86222024-02-03T17:26:21.872Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.033480824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86232024-02-03T17:26:21.872Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.033504721Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86242024-02-03T17:26:21.876Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.03425328Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86252024-02-03T17:26:21.876Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.034335642Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86262024-02-03T17:26:21.877Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.034438054Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86272024-02-03T17:26:21.877Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.034492514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86282024-02-03T17:26:21.877Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.034534045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86292024-02-03T17:26:21.877Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.034611461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86302024-02-03T17:26:21.877Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.034810989Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86312024-02-03T17:26:21.877Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.035525719Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86322024-02-03T17:26:21.877Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.035590444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86332024-02-03T17:26:21.877Z{"msg":"[0]R flush_numbers[0..12]: [148, 145, 141, 141, 143, 147, 147, 133, 143, 140, 147, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.035689044Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86342024-02-03T17:26:21.877Z{"msg":"[0]R generation[0..12]: [79, 78, 76, 76, 76, 78, 78, 72, 76, 75, 78, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.035743795Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86352024-02-03T17:26:21.877Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.035775432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86362024-02-03T17:26:21.877Z{"msg":"[1]R flush_numbers[0..12]: [148, 145, 141, 141, 143, 147, 147, 133, 143, 140, 147, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.035835661Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86372024-02-03T17:26:21.877Z{"msg":"[1]R generation[0..12]: [79, 78, 76, 76, 76, 78, 78, 72, 76, 75, 78, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.035872622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86382024-02-03T17:26:21.877Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
86392024-02-03T17:26:21.877Z,"time":"2024-02-03T17:26:21.035928101Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86402024-02-03T17:26:21.877Z{"msg":"[2]R flush_numbers[0..12]: [148, 145, 141, 141, 143, 147, 147, 133, 143, 140, 147, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.035961579Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86412024-02-03T17:26:21.877Z{"msg":"[2]R generation[0..12]: [79, 78, 76, 76, 76, 78, 78, 72, 76, 75, 78, 76]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036032163Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86422024-02-03T17:26:21.877Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036059545Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86432024-02-03T17:26:21.877Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036131314Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86442024-02-03T17:26:21.877Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036187763Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86452024-02-03T17:26:21.877Z{"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036212849Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86462024-02-03T17:26:21.877Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036245479Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"":"downstairs","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86472024-02-03T17:26:21.877Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036308356Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86482024-02-03T17:26:21.878Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036335974Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"0","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86492024-02-03T17:26:21.878Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036404815Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"1","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86502024-02-03T17:26:21.878Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036431452Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"client":"2","":"downstairs","session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86512024-02-03T17:26:21.878Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86522024-02-03T17:26:21.878Z2024-02-03T17:26:21.036492819Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86532024-02-03T17:26:21.878Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036523218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86542024-02-03T17:26:21.878Z{"msg":"36102da0-d798-47ff-9f76-56e40305fd22 is now active with session: 7f760491-2679-42d4-b4e3-ba2f02109f9f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036580958Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86552024-02-03T17:26:21.878Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036609751Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452,"session_id":"7f760491-2679-42d4-b4e3-ba2f02109f9f"}
86562024-02-03T17:26:21.878Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:21.036664133Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452}
86572024-02-03T17:26:21.878ZLoading write count information from file "/tmp/test_repair_verify.data"
86582024-02-03T17:26:21.878ZRun Repair workload
86592024-02-03T17:26:21.878Z01/30 Write block 133 len 40960 data: 7 9 9 11 10 12 9 7 7 6
86602024-02-03T17:26:21.878Z02/30 Read block 334 len 4096
86612024-02-03T17:26:21.882Z03/30 Read block 328 len 20480
86622024-02-03T17:26:21.882Z04/30 Read block 407 len 28672
86632024-02-03T17:26:21.882Z05/30 Write block 456 len 32768 data: 7 9 8 8 6 6 8 8
86642024-02-03T17:26:21.886Z06/30 Write block 116 len 12288 data: 9 10 10
86652024-02-03T17:26:21.886Z07/30 Write block 565 len 12288 data: 6 8 8
86662024-02-03T17:26:21.886Z08/30 Read block 41 len 4096
86672024-02-03T17:26:21.886Z09/30 Read block 320 len 36864
86682024-02-03T17:26:21.889Z10/30 Write block 269 len 40960 data: 10 13 14 14 14 14 14 15 12 12
86692024-02-03T17:26:21.890Z11/30 Write block 155 len 36864 data: 7 6 6 5 7 7 9 10 8
86702024-02-03T17:26:21.890Z12/30 Read block 87 len 28672
86712024-02-03T17:26:21.893Z13/30 Write block 471 len 40960 data: 5 4 3 4 4 5 8 9 9 8
86722024-02-03T17:26:21.897Z14/30 Write block 334 len 12288 data: 7 5 5
86732024-02-03T17:26:21.897Z15/30 Read block 504 len 16384
86742024-02-03T17:26:21.897Z16/30 Write block 407 len 36864 data: 8 6 7 7 8 7 7 6 2
86752024-02-03T17:26:21.900Z17/30 Read block 321 len 32768
86762024-02-03T17:26:21.900Z18/30 Read block 234 len 28672
86772024-02-03T17:26:21.904Z19/30 Write block 584 len 12288 data: 8 6 7
86782024-02-03T17:26:21.904Z20/30 Read block 262 len 24576
86792024-02-03T17:26:21.905Z21/30 Write block 524 len 20480 data: 6 7 9 10 9
86802024-02-03T17:26:21.908Z22/30 Write block 583 len 40960 data: 8 9 7 8 6 6 7 8 8 8
86812024-02-03T17:26:21.908Z23/30 Read block 501 len 4096
86822024-02-03T17:26:21.908Z24/30 Read block 546 len 28672
86832024-02-03T17:26:21.913Z25/30 Read block 193 len 4096
86842024-02-03T17:26:21.913Z26/30 Read block 44 len 8192
86852024-02-03T17:26:21.913Z27/30 Write block 113 len 20480 data: 11 12 11 10 11
86862024-02-03T17:26:21.921Z28/30 Write block 301 len 36864 data: 7 8 8 7 8 9 6 7 7
86872024-02-03T17:26:21.922Z29/30 Write block 48 len 36864 data: 5 7 8 8 8 7 4 4 5
86882024-02-03T17:26:21.922Z30/30 Write block 231 len 36864 data: 6 6 6 7 7 7 7 6 9
86892024-02-03T17:26:21.922Z----------------------------------------------------------------
86902024-02-03T17:26:21.922Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
86912024-02-03T17:26:21.922ZStates: Active Active Active
86922024-02-03T17:26:21.922ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
86932024-02-03T17:26:21.922Z 1 Acked 1000 Write 10 Done Done Done false
86942024-02-03T17:26:21.922Z 2 Acked 1001 Read 1 Done Done Done false
86952024-02-03T17:26:21.922Z 3 Acked 1002 Read 5 Done Done Done false
86962024-02-03T17:26:21.922Z 4 Acked 1003 Read 7 Done Done Done false
86972024-02-03T17:26:21.922Z 5 Acked 1004 Write 8 Sent Done Done false
86982024-02-03T17:26:21.922Z 6 Acked 1005 Write 3 Sent Done Done false
86992024-02-03T17:26:21.922Z 7 Acked 1006 Write 3 Sent Done Done false
87002024-02-03T17:26:21.922Z 8 Acked 1007 Read 1 Sent Done Done false
87012024-02-03T17:26:21.922Z 9 Acked 1008 Read 9 Sent Done Done false
87022024-02-03T17:26:21.922Z 10 Acked 1009 Write 10 Sent Done Done false
87032024-02-03T17:26:21.922Z 11 Acked 1010 Write 9 Sent Done Done false
87042024-02-03T17:26:21.922Z 12 Acked 1011 Read 7 Sent Done Done false
87052024-02-03T17:26:21.922Z 13 Acked 1012 Write 10 Sent Done Done false
87062024-02-03T17:26:21.922Z 14 Acked 1013 Write 3 Sent Done Done false
87072024-02-03T17:26:21.922Z 15 Acked 1014 Read 4 Sent Done Done false
87082024-02-03T17:26:21.923Z 16 Acked 1015 Write 9 Sent Done Done false
87092024-02-03T17:26:21.923Z 17 Acked 1016 Read 8 Sent Done Done false
87102024-02-03T17:26:21.923Z 18 Acked 1017 Read 7 Sent Done Done false
87112024-02-03T17:26:21.923Z 19 Acked 1018 Write 3 Sent Done Done false
87122024-02-03T17:26:21.923Z 20 Acked 1019 Read 6 Sent Done Done false
87132024-02-03T17:26:21.923Z 21 Acked 1020 Write 5 Sent Done Done false
87142024-02-03T17:26:21.923Z 22 Acked 1021 Write 10 Sent Done Done false
87152024-02-03T17:26:21.923Z 23 Acked 1022 Read 1 Sent Done Done false
87162024-02-03T17:26:21.923Z 24 Acked 1023 Read 7 Sent Done Done false
87172024-02-03T17:26:21.923Z 25 Acked 1024 Read 1 Sent Done Done false
87182024-02-03T17:26:21.923Z 26 Acked 1025 Read 2 Sent Done Done false
87192024-02-03T17:26:21.923Z 27 Acked 1026 Write 5 Sent Done Done false
87202024-02-03T17:26:21.923Z 28 Acked 1027 Write 9 Sent Done Done false
87212024-02-03T17:26:21.923Z 29 Acked 1028 Write 9 Sent Sent Sent false
87222024-02-03T17:26:21.923Z 30 Acked 1029 Write 9 Sent Sent Sent false
87232024-02-03T17:26:21.924Z STATES DS:0 DS:1 DS:2 TOTAL
87242024-02-03T17:26:21.924Z New 0 0 0 0
87252024-02-03T17:26:21.924Z Sent 26 2 2 30
87262024-02-03T17:26:21.924Z Done 4 28 28 60
87272024-02-03T17:26:21.924Z Skipped 0 0 0 0
87282024-02-03T17:26:21.924Z Error 0 0 0 0
87292024-02-03T17:26:21.924ZLast Flush: 0 0 0
87302024-02-03T17:26:21.924ZDownstairs last five completed:
87312024-02-03T17:26:21.924ZUpstairs last five completed: 30 29 28 27 26
87322024-02-03T17:26:21.924Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:21.081456654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1452}
87332024-02-03T17:26:21.924ZWrote out file "/tmp/test_repair_verify.data"
87342024-02-03T17:26:21.928Z
87352024-02-03T17:26:21.934ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87362024-02-03T17:26:21.966ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87372024-02-03T17:26:21.966Z 0 000-019 79 79 79 148 148 148 F F F
87382024-02-03T17:26:21.966Z 1 020-039 78 78 78 145 145 145 F F F
87392024-02-03T17:26:21.966Z 2 040-059 76 76 76 141 141 141 F T T <---
87402024-02-03T17:26:21.966Z 3 060-079 76 76 76 141 141 141 F F F
87412024-02-03T17:26:21.966Z 4 080-099 76 76 76 143 143 143 F F F
87422024-02-03T17:26:21.966Z 5 100-119 78 78 78 147 147 147 F T T <---
87432024-02-03T17:26:21.966Z 6 120-139 78 78 78 147 147 147 T T T
87442024-02-03T17:26:21.967Z 7 140-159 72 72 72 133 133 133 T T T
87452024-02-03T17:26:21.967Z 8 160-179 76 76 76 143 143 143 F T T <---
87462024-02-03T17:26:21.967Z 9 180-199 75 75 75 140 140 140 F F F
87472024-02-03T17:26:21.971Z 10 200-219 78 78 78 147 147 147 F F F
87482024-02-03T17:26:21.971Z 11 220-239 76 76 76 143 143 143 F T T <---
87492024-02-03T17:26:21.971Z 12 240-259 59 59 59 105 105 105 F F F
87502024-02-03T17:26:21.971Z 13 260-279 76 76 76 143 143 143 F T T <---
87512024-02-03T17:26:21.971Z 14 280-299 76 76 76 143 143 143 F F F
87522024-02-03T17:26:21.971Z 15 300-319 78 78 78 145 145 145 F T T <---
87532024-02-03T17:26:21.971Z 16 320-339 74 74 74 136 136 136 F T T <---
87542024-02-03T17:26:21.971Z 17 340-359 79 79 79 148 148 148 F F F
87552024-02-03T17:26:21.972Z 18 360-379 79 79 79 148 148 148 F F F
87562024-02-03T17:26:21.972Z 19 380-399 76 76 76 143 143 143 F F F
87572024-02-03T17:26:21.972Z 20 400-419 79 79 79 148 148 148 F T T <---
87582024-02-03T17:26:21.972Z 21 420-439 79 79 79 148 148 148 F F F
87592024-02-03T17:26:21.972Z 22 440-459 75 75 75 140 140 140 F T T <---
87602024-02-03T17:26:21.972Z 23 460-479 75 75 75 140 140 140 F T T <---
87612024-02-03T17:26:21.972Z 24 480-499 78 78 78 145 145 145 F T T <---
87622024-02-03T17:26:21.972Z 25 500-519 76 76 76 143 143 143 F F F
87632024-02-03T17:26:21.972Z 26 520-539 78 78 78 145 145 145 F T T <---
87642024-02-03T17:26:21.972Z 27 540-559 77 77 77 144 144 144 F F F
87652024-02-03T17:26:21.972Z 28 560-579 76 76 76 143 143 143 F T T <---
87662024-02-03T17:26:21.972Z 29 580-599 74 74 74 138 138 138 F T T <---
87672024-02-03T17:26:21.973ZMax gen: 79, Max flush: 148
87682024-02-03T17:26:21.973ZError: Difference in extent metadata found!
87692024-02-03T17:26:21.976ZA Difference in extent metadata is expected here
87702024-02-03T17:26:21.977ZOn loop 40
87712024-02-03T17:26:21.977Z
87722024-02-03T17:26:21.977ZVerifying data now
87732024-02-03T17:26:27.467ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
87742024-02-03T17:26:27.470Z157a158
87752024-02-03T17:26:27.470Z> 7,
87762024-02-03T17:26:27.470Z159,161d159
87772024-02-03T17:26:27.470Z< 5,
87782024-02-03T17:26:27.470Z< 5,
87792024-02-03T17:26:27.470Z< 4,
87802024-02-03T17:26:27.470Z162a161
87812024-02-03T17:26:27.470Z> 5,
87822024-02-03T17:26:27.471Z163a163
87832024-02-03T17:26:27.471Z> 7,
87842024-02-03T17:26:27.471Zdiff found after verify
87852024-02-03T17:26:27.476ZLoop: 40 Downstairs dump after verify (and repair):
87862024-02-03T17:26:27.501ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87872024-02-03T17:26:27.501Z 0 000-019 79 79 79 148 148 148 F F F
87882024-02-03T17:26:27.501Z 1 020-039 78 78 78 145 145 145 F F F
87892024-02-03T17:26:27.501Z 2 040-059 80 80 80 149 149 149 F F F
87902024-02-03T17:26:27.505Z 3 060-079 76 76 76 141 141 141 F F F
87912024-02-03T17:26:27.506Z 4 080-099 76 76 76 143 143 143 F F F
87922024-02-03T17:26:27.506Z 5 100-119 80 80 80 149 149 149 F F F
87932024-02-03T17:26:27.506Z 6 120-139 80 80 80 149 149 149 F F F
87942024-02-03T17:26:27.506Z 7 140-159 80 80 80 149 149 149 F F F
87952024-02-03T17:26:27.506Z 8 160-179 80 80 80 149 149 149 F F F
87962024-02-03T17:26:27.506Z 9 180-199 75 75 75 140 140 140 F F F
87972024-02-03T17:26:27.506Z 10 200-219 78 78 78 147 147 147 F F F
87982024-02-03T17:26:27.506Z 11 220-239 80 80 80 149 149 149 F F F
87992024-02-03T17:26:27.506Z 12 240-259 59 59 59 105 105 105 F F F
88002024-02-03T17:26:27.506Z 13 260-279 80 80 80 149 149 149 F F F
88012024-02-03T17:26:27.506Z 14 280-299 76 76 76 143 143 143 F F F
88022024-02-03T17:26:27.506Z 15 300-319 80 80 80 149 149 149 F F F
88032024-02-03T17:26:27.506Z 16 320-339 80 80 80 149 149 149 F F F
88042024-02-03T17:26:27.506Z 17 340-359 79 79 79 148 148 148 F F F
88052024-02-03T17:26:27.506Z 18 360-379 79 79 79 148 148 148 F F F
88062024-02-03T17:26:27.506Z 19 380-399 76 76 76 143 143 143 F F F
88072024-02-03T17:26:27.506Z 20 400-419 80 80 80 149 149 149 F F F
88082024-02-03T17:26:27.506Z 21 420-439 79 79 79 148 148 148 F F F
88092024-02-03T17:26:27.506Z 22 440-459 80 80 80 149 149 149 F F F
88102024-02-03T17:26:27.506Z 23 460-479 80 80 80 149 149 149 F F F
88112024-02-03T17:26:27.506Z 24 480-499 80 80 80 149 149 149 F F F
88122024-02-03T17:26:27.506Z 25 500-519 76 76 76 143 143 143 F F F
88132024-02-03T17:26:27.506Z 26 520-539 80 80 80 149 149 149 F F F
88142024-02-03T17:26:27.507Z 27 540-559 77 77 77 144 144 144 F F F
88152024-02-03T17:26:27.507Z 28 560-579 80 80 80 149 149 149 F F F
88162024-02-03T17:26:27.507Z 29 580-599 80 80 80 149 149 149 F F F
88172024-02-03T17:26:27.507ZMax gen: 80, Max flush: 149
88182024-02-03T17:26:27.510Z
88192024-02-03T17:26:27.510ZBegin loop 41
88202024-02-03T17:26:27.510ZDownstairs to restart: 1
88212024-02-03T17:26:27.541Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.705678961Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463}
88222024-02-03T17:26:27.541Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706139955Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463}
88232024-02-03T17:26:27.541Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706169319Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463}
88242024-02-03T17:26:27.552Z{"msg":"Crucible 8064e5a2-f925-43af-9b23-0eecbfab5644 has session id: 3784c102-529d-4672-b717-da0b005be210","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706249235Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463Crucible runtime is spawned
88252024-02-03T17:26:27.552Z,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88262024-02-03T17:26:27.553Z{"msg":"Upstairs opts: Upstairs UUID: 8064e5a2-f925-43af-9b23-0eecbfab5644, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.70640165Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88272024-02-03T17:26:27.553Z{"msg":"Crucible stats registered with UUID: 8064e5a2-f925-43af-9b23-0eecbfab5644","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706479835Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88282024-02-03T17:26:27.553Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706505885Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463}
88292024-02-03T17:26:27.553Z{"msg":"8064e5a2-f925-43af-9b23-0eecbfab5644 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706708934Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88302024-02-03T17:26:27.553Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706790762Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88312024-02-03T17:26:27.553Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706834851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88322024-02-03T17:26:27.553Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.706874297Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88332024-02-03T17:26:27.553Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.70691358Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88342024-02-03T17:26:27.553Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.70698836Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88352024-02-03T17:26:27.553Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.707033388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88362024-02-03T17:26:27.553Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.707073487Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88372024-02-03T17:26:27.553Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.707112824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88382024-02-03T17:26:27.553Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.7072081Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88392024-02-03T17:26:27.553Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.707346136Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88402024-02-03T17:26:27.553Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.707451932Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88412024-02-03T17:26:27.553Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.707501029Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88422024-02-03T17:26:27.553Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.708716196Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88432024-02-03T17:26:27.553Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.708784186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88442024-02-03T17:26:27.553Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.708811432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88452024-02-03T17:26:27.553Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.708834804Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88462024-02-03T17:26:27.553Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.709786997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88472024-02-03T17:26:27.553Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.709877579Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88482024-02-03T17:26:27.553Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.710283904Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88492024-02-03T17:26:27.554Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.710339635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88502024-02-03T17:26:27.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.710897092Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88512024-02-03T17:26:27.554Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.710968128Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88522024-02-03T17:26:27.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.710996657Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88532024-02-03T17:26:27.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.711017912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88542024-02-03T17:26:27.554Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.711060678Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88552024-02-03T17:26:27.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.711089929Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88562024-02-03T17:26:27.554Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.711444472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88572024-02-03T17:26:27.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712171988Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88582024-02-03T17:26:27.554Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712231509Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88592024-02-03T17:26:27.554Z{"msg":"[0]R flush_numbers[0..12]: [148, 145, 149, 141, 143, 149, 149, 149, 149, 140, 147, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712259091Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88602024-02-03T17:26:27.554Z{"msg":"[0]R generation[0..12]: [79, 78, 80, 76, 76, 80, 80, 80, 80, 75, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712282871Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88612024-02-03T17:26:27.554Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.71230697Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88622024-02-03T17:26:27.554Z{"msg":"[1]R flush_numbers[0..12]: [148, 145, 149, 141, 143, 149, 149, 149, 149, 140, 147, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712331252Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88632024-02-03T17:26:27.554Z{"msg":"[1]R generation[0..12]: [79, 78, 80, 76, 76, 80, 80, 80, 80, 75, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712373158Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88642024-02-03T17:26:27.554Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712399608Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88652024-02-03T17:26:27.554Z{"msg":"[2]R flush_numbers[0..12]: [148, 145, 149, 141, 143, 149, 149, 149, 149, 140, 147, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712423414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88662024-02-03T17:26:27.554Z{"msg":"[2]R generation[0..12]: [79, 78, 80, 76, 76, 80, 80, 80, 80, 75, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712464785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88672024-02-03T17:26:27.554Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712490748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88682024-02-03T17:26:27.554Z{"msg":"Max found gen is 81","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712513023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88692024-02-03T17:26:27.554Z{"msg":"Generation requested: 82 >= found:81","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712535238Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88702024-02-03T17:26:27.554Z{"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712556255Z","Wait for a query_work_queue command to finish before sending IO
88712024-02-03T17:26:27.554Zhostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88722024-02-03T17:26:27.554Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712581313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88732024-02-03T17:26:27.555Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712604644Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88742024-02-03T17:26:27.555Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712624586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88752024-02-03T17:26:27.555Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712646848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88762024-02-03T17:26:27.555Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712687165Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"3784c102-529d-4672-b717-da0b005be210"}
88772024-02-03T17:26:27.555Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712711557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88782024-02-03T17:26:27.555Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712732548Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88792024-02-03T17:26:27.555Z{"msg":"8064e5a2-f925-43af-9b23-0eecbfab5644 is now active with session: 3784c102-529d-4672-b717-da0b005be210","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712792232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88802024-02-03T17:26:27.555Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.712814329Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463,"session_id":"3784c102-529d-4672-b717-da0b005be210"}
88812024-02-03T17:26:27.555Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:26.71283485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463}
88822024-02-03T17:26:27.555ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88832024-02-03T17:26:27.555ZLoading write count information from file "/tmp/test_repair_verify.data"
88842024-02-03T17:26:27.555ZRun Repair workload
88852024-02-03T17:26:27.555Z01/30 Write block 109 len 36864 data: 10 10 10 11 12 13 12 11 12
88862024-02-03T17:26:27.559Z02/30 Read block 559 len 20480
88872024-02-03T17:26:27.559Z03/30 Read block 518 len 36864
88882024-02-03T17:26:27.559Z04/30 Read block 371 len 28672
88892024-02-03T17:26:27.559Z05/30 Write block 50 len 24576 data: 9 9 9 8 5 5
88902024-02-03T17:26:27.559Z06/30 Read block 529 len 32768
88912024-02-03T17:26:27.562Z07/30 Read block 382 len 36864
88922024-02-03T17:26:27.567Z08/30 Write block 515 len 16384 data: 5 3 4 3
88932024-02-03T17:26:27.567Z09/30 Write block 267 len 24576 data: 5 8 11 14 15 15
88942024-02-03T17:26:27.567Z10/30 Write block 177 len 36864 data: 8 7 5 7 9 9 7 5 6
88952024-02-03T17:26:27.567Z11/30 Read block 375 len 32768
88962024-02-03T17:26:27.570Z12/30 Read block 144 len 36864
88972024-02-03T17:26:27.573Z13/30 Write block 457 len 16384 data: 10 9 9 7
88982024-02-03T17:26:27.573Z14/30 Write block 63 len 8192 data: 7 7
88992024-02-03T17:26:27.573Z15/30 Flush
89002024-02-03T17:26:27.576Z16/30 Write block 508 len 12288 data: 12 12 9
89012024-02-03T17:26:27.579Z17/30 Flush
89022024-02-03T17:26:27.582Z18/30 Write block 86 len 28672 data: 6 7 6 7 8 7 7
89032024-02-03T17:26:27.582Z19/30 Read block 210 len 24576
89042024-02-03T17:26:27.585Z20/30 Read block 59 len 24576
89052024-02-03T17:26:27.585Z21/30 Write block 376 len 16384 data: 6 6 6 5
89062024-02-03T17:26:27.585Z22/30 Read block 306 len 28672
89072024-02-03T17:26:27.589Z23/30 Read block 285 len 32768
89082024-02-03T17:26:27.589Z24/30 Read block 483 len 28672
89092024-02-03T17:26:27.592Z25/30 Write block 83 len 20480 data: 9 10 7 7 8
89102024-02-03T17:26:27.592Z26/30 Read block 216 len 40960
89112024-02-03T17:26:27.595Z27/30 Write block 99 len 12288 data: 4 5 5
89122024-02-03T17:26:27.595Z28/30 Write block 228 len 40960 data: 10 6 5 7 7 7 8 8 8 8
89132024-02-03T17:26:27.599Z29/30 Write block 128 len 20480 data: 10 12 12 11 10
89142024-02-03T17:26:27.599Z30/30 Write block 196 len 16384 data: 6 5 5 6
89152024-02-03T17:26:27.599Z----------------------------------------------------------------
89162024-02-03T17:26:27.599Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89172024-02-03T17:26:27.599ZStates: Active Active Active
89182024-02-03T17:26:27.599ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89192024-02-03T17:26:27.599Z 1 Acked 1000 Write 9 Done Done Done false
89202024-02-03T17:26:27.599Z 2 Acked 1001 Read 5 Done Done Done false
89212024-02-03T17:26:27.599Z 3 Acked 1002 Read 9 Done Sent Done false
89222024-02-03T17:26:27.599Z 4 Acked 1003 Read 7 Done Sent Done false
89232024-02-03T17:26:27.599Z 5 Acked 1004 Write 6 Done Sent Done false
89242024-02-03T17:26:27.599Z 6 Acked 1005 Read 8 Done Sent Done false
89252024-02-03T17:26:27.599Z 7 Acked 1006 Read 9 Done Sent Done false
89262024-02-03T17:26:27.599Z 8 Acked 1007 Write 4 Done Sent Done false
89272024-02-03T17:26:27.599Z 9 Acked 1008 Write 6 Done Sent Done false
89282024-02-03T17:26:27.599Z 10 Acked 1009 Write 9 Done Sent Done false
89292024-02-03T17:26:27.599Z 11 Acked 1010 Read 8 Done Sent Done false
89302024-02-03T17:26:27.600Z 12 Acked 1011 Read 9 Done Sent Done false
89312024-02-03T17:26:27.600Z 13 Acked 1012 Write 4 Done Sent Done false
89322024-02-03T17:26:27.600Z 14 Acked 1013 Write 2 Done Sent Done false
89332024-02-03T17:26:27.600Z 15 Acked 1014 Flush 0 Done Sent Done false
89342024-02-03T17:26:27.600Z 16 Acked 1015 Write 3 Done Sent Done false
89352024-02-03T17:26:27.600Z 17 Acked 1016 Flush 0 Done Sent Done false
89362024-02-03T17:26:27.600Z 18 Acked 1017 Write 7 Done Sent Done false
89372024-02-03T17:26:27.600Z 19 Acked 1018 Read 6 Done Sent Done false
89382024-02-03T17:26:27.600Z 20 Acked 1019 Read 6 Done Sent Done false
89392024-02-03T17:26:27.600Z 21 Acked 1020 Write 4 Done Sent Done false
89402024-02-03T17:26:27.600Z 22 Acked 1021 Read 7 Done Sent Done false
89412024-02-03T17:26:27.600Z 23 Acked 1022 Read 8 Done Sent Done false
89422024-02-03T17:26:27.600Z 24 Acked 1023 Read 7 Done Sent Done false
89432024-02-03T17:26:27.600Z 25 Acked 1024 Write 5 Done Sent Done false
89442024-02-03T17:26:27.600Z 26 Acked 1025 Read 10 Done Sent Done false
89452024-02-03T17:26:27.600Z 27 Acked 1026 Write 3 Done Sent Done false
89462024-02-03T17:26:27.600Z 28 Acked 1027 Write 10 Done Sent Done false
89472024-02-03T17:26:27.600Z 29 Acked 1028 Write 5 Sent Sent Sent false
89482024-02-03T17:26:27.600Z 30 Acked 1029 Write 4 Sent Sent Sent false
89492024-02-03T17:26:27.600Z STATES DS:0 DS:1 DS:2 TOTAL
89502024-02-03T17:26:27.600Z New 0 0 0 0
89512024-02-03T17:26:27.600Z Sent 2 28 2 32
89522024-02-03T17:26:27.600Z Done 28 2 28 58
89532024-02-03T17:26:27.600Z Skipped 0 0 0 0
89542024-02-03T17:26:27.600Z Error 0 0 0 0
89552024-02-03T17:26:27.601ZLast Flush: 1016 0 1016
89562024-02-03T17:26:27.601ZDownstairs last five completed:
89572024-02-03T17:26:27.601ZUpstairs last five completed: 30 29 28 27 26
89582024-02-03T17:26:27.601Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:26.763650302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1463}
89592024-02-03T17:26:27.605ZWrote out file "/tmp/test_repair_verify.data"
89602024-02-03T17:26:27.608Z
89612024-02-03T17:26:27.614ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89622024-02-03T17:26:27.642ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89632024-02-03T17:26:27.642Z 0 000-019 79 79 79 148 148 148 F F F
89642024-02-03T17:26:27.642Z 1 020-039 78 78 78 145 145 145 F F F
89652024-02-03T17:26:27.642Z 2 040-059 82 80 82 150 149 150 F F F <---
89662024-02-03T17:26:27.642Z 3 060-079 82 76 82 150 141 150 F F F <---
89672024-02-03T17:26:27.645Z 4 080-099 76 76 76 143 143 143 T F T <---
89682024-02-03T17:26:27.645Z 5 100-119 82 80 82 150 149 150 T T T <---
89692024-02-03T17:26:27.645Z 6 120-139 80 80 80 149 149 149 T F T <---
89702024-02-03T17:26:27.646Z 7 140-159 80 80 80 149 149 149 F F F
89712024-02-03T17:26:27.646Z 8 160-179 82 80 82 150 149 150 F F F <---
89722024-02-03T17:26:27.646Z 9 180-199 82 75 82 150 140 150 T F T <---
89732024-02-03T17:26:27.646Z 10 200-219 78 78 78 147 147 147 F F F
89742024-02-03T17:26:27.646Z 11 220-239 80 80 80 149 149 149 T F T <---
89752024-02-03T17:26:27.646Z 12 240-259 59 59 59 105 105 105 F F F
89762024-02-03T17:26:27.646Z 13 260-279 82 80 82 150 149 150 F F F <---
89772024-02-03T17:26:27.646Z 14 280-299 76 76 76 143 143 143 F F F
89782024-02-03T17:26:27.646Z 15 300-319 80 80 80 149 149 149 F F F
89792024-02-03T17:26:27.646Z 16 320-339 80 80 80 149 149 149 F F F
89802024-02-03T17:26:27.646Z 17 340-359 79 79 79 148 148 148 F F F
89812024-02-03T17:26:27.646Z 18 360-379 79 79 79 148 148 148 T F T <---
89822024-02-03T17:26:27.646Z 19 380-399 76 76 76 143 143 143 F F F
89832024-02-03T17:26:27.646Z 20 400-419 80 80 80 149 149 149 F F F
89842024-02-03T17:26:27.646Z 21 420-439 79 79 79 148 148 148 F F F
89852024-02-03T17:26:27.646Z 22 440-459 82 80 82 150 149 150 F F F <---
89862024-02-03T17:26:27.646Z 23 460-479 82 80 82 150 149 150 F F F <---
89872024-02-03T17:26:27.646Z 24 480-499 80 80 80 149 149 149 F F F
89882024-02-03T17:26:27.646Z 25 500-519 82 76 82 151 143 151 F F F <---
89892024-02-03T17:26:27.646Z 26 520-539 80 80 80 149 149 149 F F F
89902024-02-03T17:26:27.646Z 27 540-559 77 77 77 144 144 144 F F F
89912024-02-03T17:26:27.646Z 28 560-579 80 80 80 149 149 149 F F F
89922024-02-03T17:26:27.646Z 29 580-599 80 80 80 149 149 149 F F F
89932024-02-03T17:26:27.646ZMax gen: 82, Max flush: 151
89942024-02-03T17:26:27.647ZError: Difference in extent metadata found!
89952024-02-03T17:26:27.650ZA Difference in extent metadata is expected here
89962024-02-03T17:26:27.650ZOn loop 41
89972024-02-03T17:26:27.650Z
89982024-02-03T17:26:27.653ZVerifying data now
89992024-02-03T17:26:32.797ZNo change after verify
90002024-02-03T17:26:32.797ZLoop: 41 Downstairs dump after verify (and repair):
90012024-02-03T17:26:32.825ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90022024-02-03T17:26:32.825Z 0 000-019 79 79 79 148 148 148 F F F
90032024-02-03T17:26:32.825Z 1 020-039 78 78 78 145 145 145 F F F
90042024-02-03T17:26:32.825Z 2 040-059 82 82 82 150 150 150 F F F
90052024-02-03T17:26:32.825Z 3 060-079 82 82 82 150 150 150 F F F
90062024-02-03T17:26:32.829Z 4 080-099 83 83 83 152 152 152 F F F
90072024-02-03T17:26:32.829Z 5 100-119 83 83 83 152 152 152 F F F
90082024-02-03T17:26:32.829Z 6 120-139 83 83 83 152 152 152 F F F
90092024-02-03T17:26:32.829Z 7 140-159 80 80 80 149 149 149 F F F
90102024-02-03T17:26:32.830Z 8 160-179 82 82 82 150 150 150 F F F
90112024-02-03T17:26:32.830Z 9 180-199 83 83 83 152 152 152 F F F
90122024-02-03T17:26:32.830Z 10 200-219 78 78 78 147 147 147 F F F
90132024-02-03T17:26:32.830Z 11 220-239 83 83 83 152 152 152 F F F
90142024-02-03T17:26:32.830Z 12 240-259 59 59 59 105 105 105 F F F
90152024-02-03T17:26:32.830Z 13 260-279 82 82 82 150 150 150 F F F
90162024-02-03T17:26:32.830Z 14 280-299 76 76 76 143 143 143 F F F
90172024-02-03T17:26:32.830Z 15 300-319 80 80 80 149 149 149 F F F
90182024-02-03T17:26:32.830Z 16 320-339 80 80 80 149 149 149 F F F
90192024-02-03T17:26:32.830Z 17 340-359 79 79 79 148 148 148 F F F
90202024-02-03T17:26:32.830Z 18 360-379 83 83 83 152 152 152 F F F
90212024-02-03T17:26:32.830Z 19 380-399 76 76 76 143 143 143 F F F
90222024-02-03T17:26:32.830Z 20 400-419 80 80 80 149 149 149 F F F
90232024-02-03T17:26:32.831Z 21 420-439 79 79 79 148 148 148 F F F
90242024-02-03T17:26:32.831Z 22 440-459 82 82 82 150 150 150 F F F
90252024-02-03T17:26:32.831Z 23 460-479 82 82 82 150 150 150 F F F
90262024-02-03T17:26:32.831Z 24 480-499 80 80 80 149 149 149 F F F
90272024-02-03T17:26:32.831Z 25 500-519 82 82 82 151 151 151 F F F
90282024-02-03T17:26:32.831Z 26 520-539 80 80 80 149 149 149 F F F
90292024-02-03T17:26:32.831Z 27 540-559 77 77 77 144 144 144 F F F
90302024-02-03T17:26:32.831Z 28 560-579 80 80 80 149 149 149 F F F
90312024-02-03T17:26:32.831Z 29 580-599 80 80 80 149 149 149 F F F
90322024-02-03T17:26:32.831ZMax gen: 83, Max flush: 152
90332024-02-03T17:26:32.831Z
90342024-02-03T17:26:32.831ZBegin loop 42
90352024-02-03T17:26:32.831ZDownstairs to restart: 2
90362024-02-03T17:26:32.862Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.026942018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471}
90372024-02-03T17:26:32.862Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.02756009Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471}
90382024-02-03T17:26:32.862Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.027608459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471}
90392024-02-03T17:26:32.867Z{"msg":"Crucible afe89bc4-3f88-44cb-a910-57ce74fd3b06 has session id: 01dde863-7462-489e-9c5d-0df03d4ab51a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.027726452Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90402024-02-03T17:26:32.867Z{"msg":"Upstairs opts: Upstairs UUID: afe89bc4-3f88-44cb-a910-57ce74fd3b06, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.027788307Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90412024-02-03T17:26:32.867Z{"msg"Crucible runtime is spawned
90422024-02-03T17:26:32.867Z:"Crucible stats registered with UUID: afe89bc4-3f88-44cb-a910-57ce74fd3b06","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.027864994Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90432024-02-03T17:26:32.867Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.027955784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471}
90442024-02-03T17:26:32.867Z{"msg":"afe89bc4-3f88-44cb-a910-57ce74fd3b06 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028184659Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90452024-02-03T17:26:32.867Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028312283Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90462024-02-03T17:26:32.867Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.02835796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90472024-02-03T17:26:32.867Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028403312Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90482024-02-03T17:26:32.867Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028445886Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90492024-02-03T17:26:32.868Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028610129Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90502024-02-03T17:26:32.868Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028697244Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90512024-02-03T17:26:32.868Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028742871Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90522024-02-03T17:26:32.868Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028813432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90532024-02-03T17:26:32.868Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028859871Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90542024-02-03T17:26:32.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.028953484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90552024-02-03T17:26:32.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.029001575Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90562024-02-03T17:26:32.868Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.029044501Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90572024-02-03T17:26:32.868Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.030167235Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90582024-02-03T17:26:32.868Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.030230693Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90592024-02-03T17:26:32.868Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.03025717Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90602024-02-03T17:26:32.868Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.030357351Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90612024-02-03T17:26:32.869Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.031104828Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90622024-02-03T17:26:32.869Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.031265026Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90632024-02-03T17:26:32.869Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.031400518Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90642024-02-03T17:26:32.869Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.03144219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90652024-02-03T17:26:32.873Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.032149488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90662024-02-03T17:26:32.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.032412524Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90672024-02-03T17:26:32.874Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.032473925Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90682024-02-03T17:26:32.874Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.032500462Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90692024-02-03T17:26:32.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.032541215Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90702024-02-03T17:26:32.874Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.032566024Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90712024-02-03T17:26:32.874Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.032607731Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90722024-02-03T17:26:32.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033057232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90732024-02-03T17:26:32.874Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033143424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90742024-02-03T17:26:32.874Z{"msg":"[0]R flush_numbers[0..12]: [148, 145, 150, 150, 152, 152, 152, 149, 150, 152, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033191943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90752024-02-03T17:26:32.874Z{"msg":"[0]R generation[0..12]: [79, 78, 82, 82, 83, 83, 83, 80, 82, 83, 78, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033263776Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90762024-02-03T17:26:32.874Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033336684Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90772024-02-03T17:26:32.875Z{"msg":"[1]R flush_numbers[0..12]: [148, 145, 150, 150, 152, 152, 152, 149, 150, 152, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033413005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90782024-02-03T17:26:32.875Z{"msg":"[1]R generation[0..12]: [79, 78, 82, 82, 83, 83, 83, 80, 82, 83, 78, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033458971Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90792024-02-03T17:26:32.875ZWait for a query_work_queue command to finish before sending IO
90802024-02-03T17:26:32.875Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033543903Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90812024-02-03T17:26:32.875Z{"msg":"[2]R flush_numbers[0..12]: [148, 145, 150, 150, 152, 152, 152, 149, 150, 152, 147, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033584749Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90822024-02-03T17:26:32.875Z{"msg":"[2]R generation[0..12]: [79, 78, 82, 82, 83, 83, 83, 80, 82, 83, 78, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033609179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90832024-02-03T17:26:32.875Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033632592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90842024-02-03T17:26:32.875Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033689407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90852024-02-03T17:26:32.875Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033714109Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90862024-02-03T17:26:32.875Z{"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033736969Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90872024-02-03T17:26:32.875Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.03375844Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90882024-02-03T17:26:32.875Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033781026Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90892024-02-03T17:26:32.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033812247Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90902024-02-03T17:26:32.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033847834Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90912024-02-03T17:26:32.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.03389719Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90922024-02-03T17:26:32.876Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033922381Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90932024-02-03T17:26:32.876Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033962787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90942024-02-03T17:26:32.876Z{"msg":"afe89bc4-3f88-44cb-a910-57ce74fd3b06 is now active with session: 01dde863-7462-489e-9c5d-0df03d4ab51a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.033986701Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90952024-02-03T17:26:32.876Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.034006443Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471,"session_id":"01dde863-7462-489e-9c5d-0df03d4ab51a"}
90962024-02-03T17:26:32.876Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:32.034066817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471}
90972024-02-03T17:26:32.876ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90982024-02-03T17:26:32.876ZLoading write count information from file "/tmp/test_repair_verify.data"
90992024-02-03T17:26:32.876ZRun Repair workload
91002024-02-03T17:26:32.876Z01/30 Write block 498 len 28672 data: 8 8 9 9 9 9 9
91012024-02-03T17:26:32.881Z02/30 Write block 48 len 36864 data: 6 8 10 10 10 9 6 6 6
91022024-02-03T17:26:32.881Z03/30 Read block 555 len 24576
91032024-02-03T17:26:32.881Z04/30 Read block 476 len 8192
91042024-02-03T17:26:32.881Z05/30 Write block 294 len 28672 data: 8 6 6 7 6 4 6
91052024-02-03T17:26:32.881Z06/30 Write block 347 len 4096 data: 7
91062024-02-03T17:26:32.881Z07/30 Read block 174 len 16384
91072024-02-03T17:26:32.896Z08/30 Write block 221 len 28672 data: 12 12 9 10 11 9 12
91082024-02-03T17:26:32.896Z09/30 Read block 227 len 4096
91092024-02-03T17:26:32.896Z10/30 Read block 307 len 40960
91102024-02-03T17:26:32.901Z11/30 Write block 175 len 20480 data: 6 6 9 8 6
91112024-02-03T17:26:32.901Z12/30 Write block 174 len 28672 data: 6 7 7 10 9 7 8
91122024-02-03T17:26:32.901Z13/30 Read block 44 len 4096
91132024-02-03T17:26:32.901Z14/30 Read block 264 len 4096
91142024-02-03T17:26:32.901Z15/30 Write block 465 len 16384 data: 10 7 8 7
91152024-02-03T17:26:32.901Z16/30 Read block 596 len 8192
91162024-02-03T17:26:32.902Z17/30 Read block 417 len 16384
91172024-02-03T17:26:32.902Z18/30 Write block 265 len 12288 data: 6 8 6
91182024-02-03T17:26:32.902Z19/30 Read block 424 len 4096
91192024-02-03T17:26:32.902Z20/30 Write block 417 len 24576 data: 6 8 8 8 6 5
91202024-02-03T17:26:32.902Z21/30 Write block 250 len 4096 data: 11
91212024-02-03T17:26:32.902Z22/30 Read block 46 len 28672
91222024-02-03T17:26:32.902Z23/30 Read block 377 len 12288
91232024-02-03T17:26:32.902Z24/30 Read block 131 len 36864
91242024-02-03T17:26:32.906Z25/30 Flush
91252024-02-03T17:26:32.909Z26/30 Write block 535 len 8192 data: 7 7
91262024-02-03T17:26:32.909Z27/30 Write block 467 len 8192 data: 9 8
91272024-02-03T17:26:32.909Z28/30 Read block 40 len 8192
91282024-02-03T17:26:32.913Z29/30 Write block 56 len 20480 data: 7 5 6 7 6
91292024-02-03T17:26:32.913Z30/30 Write block 137 len 20480 data: 11 13 10 8 8
91302024-02-03T17:26:32.914Z----------------------------------------------------------------
91312024-02-03T17:26:32.914Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91322024-02-03T17:26:32.914ZStates: Active Active Active
91332024-02-03T17:26:32.914ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91342024-02-03T17:26:32.914Z 1 Acked 1000 Write 7 Done Done Done false
91352024-02-03T17:26:32.914Z 2 Acked 1001 Write 9 Done Done Done false
91362024-02-03T17:26:32.914Z 3 Acked 1002 Read 6 Done Done Sent false
91372024-02-03T17:26:32.914Z 4 Acked 1003 Read 2 Done Done Sent false
91382024-02-03T17:26:32.914Z 5 Acked 1004 Write 7 Done Done Sent false
91392024-02-03T17:26:32.914Z 6 Acked 1005 Write 1 Done Done Sent false
91402024-02-03T17:26:32.914Z 7 Acked 1006 Read 4 Done Done Sent false
91412024-02-03T17:26:32.914Z 8 Acked 1007 Write 7 Done Done Sent false
91422024-02-03T17:26:32.914Z 9 Acked 1008 Read 1 Done Done Sent false
91432024-02-03T17:26:32.915Z 10 Acked 1009 Read 10 Done Done Sent false
91442024-02-03T17:26:32.915Z 11 Acked 1010 Write 5 Done Done Sent false
91452024-02-03T17:26:32.915Z 12 Acked 1011 Write 7 Done Done Sent false
91462024-02-03T17:26:32.915Z 13 Acked 1012 Read 1 Done Done Sent false
91472024-02-03T17:26:32.915Z 14 Acked 1013 Read 1 Done Done Sent false
91482024-02-03T17:26:32.915Z 15 Acked 1014 Write 4 Done Done Sent false
91492024-02-03T17:26:32.915Z 16 Acked 1015 Read 2 Done Done Sent false
91502024-02-03T17:26:32.915Z 17 Acked 1016 Read 4 Done Done Sent false
91512024-02-03T17:26:32.915Z 18 Acked 1017 Write 3 Done Done Sent false
91522024-02-03T17:26:32.915Z 19 Acked 1018 Read 1 Done Done Sent false
91532024-02-03T17:26:32.915Z 20 Acked 1019 Write 6 Done Done Sent false
91542024-02-03T17:26:32.915Z 21 Acked 1020 Write 1 Done Done Sent false
91552024-02-03T17:26:32.916Z 22 Acked 1021 Read 7 Done Done Sent false
91562024-02-03T17:26:32.916Z 23 Acked 1022 Read 3 Done Done Sent false
91572024-02-03T17:26:32.916Z 24 Acked 1023 Read 9 Done Done Sent false
91582024-02-03T17:26:32.916Z 25 Acked 1024 Flush 0 Done Done Sent false
91592024-02-03T17:26:32.916Z 26 Acked 1025 Write 2 Done Done Sent false
91602024-02-03T17:26:32.916Z 27 Acked 1026 Write 2 Done Done Sent false
91612024-02-03T17:26:32.916Z 28 Acked 1027 Read 2 Done Done Sent false
91622024-02-03T17:26:32.916Z 29 Acked 1028 Write 5 Done Sent Sent false
91632024-02-03T17:26:32.916Z 30 Acked 1029 Write 5 Sent Sent Sent false
91642024-02-03T17:26:32.916Z STATES DS:0 DS:1 DS:2 TOTAL
91652024-02-03T17:26:32.916Z New 0 0 0 0
91662024-02-03T17:26:32.916Z Sent 1 2 28 31
91672024-02-03T17:26:32.917Z Done 29 28 2 59
91682024-02-03T17:26:32.917Z Skipped 0 0 0 0
91692024-02-03T17:26:32.917Z Error 0 0 0 0
91702024-02-03T17:26:32.917ZLast Flush: 1024 1024 0
91712024-02-03T17:26:32.917ZDownstairs last five completed:
91722024-02-03T17:26:32.917ZUpstairs last five completed: 30 29 28 27 26
91732024-02-03T17:26:32.917Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:32.078005414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1471}
91742024-02-03T17:26:32.917ZWrote out file "/tmp/test_repair_verify.data"
91752024-02-03T17:26:32.921Z
91762024-02-03T17:26:32.928ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91772024-02-03T17:26:32.957ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91782024-02-03T17:26:32.957Z 0 000-019 79 79 79 148 148 148 F F F
91792024-02-03T17:26:32.957Z 1 020-039 78 78 78 145 145 145 F F F
91802024-02-03T17:26:32.957Z 2 040-059 84 84 82 153 153 150 T T T <---
91812024-02-03T17:26:32.958Z 3 060-079 82 82 82 150 150 150 T T F <---
91822024-02-03T17:26:32.958Z 4 080-099 83 83 83 152 152 152 F F F
91832024-02-03T17:26:32.962Z 5 100-119 83 83 83 152 152 152 F F F
91842024-02-03T17:26:32.962Z 6 120-139 83 83 83 152 152 152 T T F <---
91852024-02-03T17:26:32.962Z 7 140-159 80 80 80 149 149 149 T T F <---
91862024-02-03T17:26:32.962Z 8 160-179 84 84 82 153 153 150 F F F <---
91872024-02-03T17:26:32.962Z 9 180-199 84 84 83 153 153 152 F F F <---
91882024-02-03T17:26:32.962Z 10 200-219 78 78 78 147 147 147 F F F
91892024-02-03T17:26:32.962Z 11 220-239 84 84 83 153 153 152 F F F <---
91902024-02-03T17:26:32.962Z 12 240-259 84 84 59 153 153 105 F F F <---
91912024-02-03T17:26:32.962Z 13 260-279 84 84 82 153 153 150 F F F <---
91922024-02-03T17:26:32.962Z 14 280-299 84 84 76 153 153 143 F F F <---
91932024-02-03T17:26:32.962Z 15 300-319 84 84 80 153 153 149 F F F <---
91942024-02-03T17:26:32.962Z 16 320-339 80 80 80 149 149 149 F F F
91952024-02-03T17:26:32.962Z 17 340-359 84 84 79 153 153 148 F F F <---
91962024-02-03T17:26:32.962Z 18 360-379 83 83 83 152 152 152 F F F
91972024-02-03T17:26:32.962Z 19 380-399 76 76 76 143 143 143 F F F
91982024-02-03T17:26:32.962Z 20 400-419 84 84 80 153 153 149 F F F <---
91992024-02-03T17:26:32.962Z 21 420-439 84 84 79 153 153 148 F F F <---
92002024-02-03T17:26:32.962Z 22 440-459 82 82 82 150 150 150 F F F
92012024-02-03T17:26:32.963Z 23 460-479 84 84 82 153 153 150 T T F <---
92022024-02-03T17:26:32.963Z 24 480-499 84 84 80 153 153 149 F F T <---
92032024-02-03T17:26:32.963Z 25 500-519 84 84 82 153 153 151 F F T <---
92042024-02-03T17:26:32.963Z 26 520-539 80 80 80 149 149 149 T T F <---
92052024-02-03T17:26:32.963Z 27 540-559 77 77 77 144 144 144 F F F
92062024-02-03T17:26:32.963Z 28 560-579 80 80 80 149 149 149 F F F
92072024-02-03T17:26:32.963Z 29 580-599 80 80 80 149 149 149 F F F
92082024-02-03T17:26:32.963ZMax gen: 84, Max flush: 153
92092024-02-03T17:26:32.963ZError: Difference in extent metadata found!
92102024-02-03T17:26:32.968ZA Difference in extent metadata is expected here
92112024-02-03T17:26:32.968ZOn loop 42
92122024-02-03T17:26:32.968Z
92132024-02-03T17:26:32.968ZVerifying data now
92142024-02-03T17:26:39.495ZNo change after verify
92152024-02-03T17:26:39.495ZLoop: 42 Downstairs dump after verify (and repair):
92162024-02-03T17:26:39.520ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92172024-02-03T17:26:39.520Z 0 000-019 79 79 79 148 148 148 F F F
92182024-02-03T17:26:39.520Z 1 020-039 78 78 78 145 145 145 F F F
92192024-02-03T17:26:39.520Z 2 040-059 85 85 85 154 154 154 F F F
92202024-02-03T17:26:39.520Z 3 060-079 85 85 85 154 154 154 F F F
92212024-02-03T17:26:39.524Z 4 080-099 83 83 83 152 152 152 F F F
92222024-02-03T17:26:39.524Z 5 100-119 83 83 83 152 152 152 F F F
92232024-02-03T17:26:39.524Z 6 120-139 85 85 85 154 154 154 F F F
92242024-02-03T17:26:39.524Z 7 140-159 85 85 85 154 154 154 F F F
92252024-02-03T17:26:39.524Z 8 160-179 84 84 84 153 153 153 F F F
92262024-02-03T17:26:39.524Z 9 180-199 84 84 84 153 153 153 F F F
92272024-02-03T17:26:39.524Z 10 200-219 78 78 78 147 147 147 F F F
92282024-02-03T17:26:39.524Z 11 220-239 84 84 84 153 153 153 F F F
92292024-02-03T17:26:39.524Z 12 240-259 84 84 84 153 153 153 F F F
92302024-02-03T17:26:39.524Z 13 260-279 84 84 84 153 153 153 F F F
92312024-02-03T17:26:39.524Z 14 280-299 84 84 84 153 153 153 F F F
92322024-02-03T17:26:39.524Z 15 300-319 84 84 84 153 153 153 F F F
92332024-02-03T17:26:39.525Z 16 320-339 80 80 80 149 149 149 F F F
92342024-02-03T17:26:39.525Z 17 340-359 84 84 84 153 153 153 F F F
92352024-02-03T17:26:39.525Z 18 360-379 83 83 83 152 152 152 F F F
92362024-02-03T17:26:39.525Z 19 380-399 76 76 76 143 143 143 F F F
92372024-02-03T17:26:39.525Z 20 400-419 84 84 84 153 153 153 F F F
92382024-02-03T17:26:39.525Z 21 420-439 84 84 84 153 153 153 F F F
92392024-02-03T17:26:39.525Z 22 440-459 82 82 82 150 150 150 F F F
92402024-02-03T17:26:39.525Z 23 460-479 85 85 85 154 154 154 F F F
92412024-02-03T17:26:39.525Z 24 480-499 84 84 84 153 153 153 F F F
92422024-02-03T17:26:39.525Z 25 500-519 84 84 84 153 153 153 F F F
92432024-02-03T17:26:39.525Z 26 520-539 85 85 85 154 154 154 F F F
92442024-02-03T17:26:39.525Z 27 540-559 77 77 77 144 144 144 F F F
92452024-02-03T17:26:39.525Z 28 560-579 80 80 80 149 149 149 F F F
92462024-02-03T17:26:39.525Z 29 580-599 80 80 80 149 149 149 F F F
92472024-02-03T17:26:39.525ZMax gen: 85, Max flush: 154
92482024-02-03T17:26:39.532Z
92492024-02-03T17:26:39.532ZBegin loop 43
92502024-02-03T17:26:39.532ZDownstairs to restart: 0
92512024-02-03T17:26:39.561Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.726166467Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479}
92522024-02-03T17:26:39.561Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.726688718Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479}
92532024-02-03T17:26:39.561Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.726732784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479}
92542024-02-03T17:26:39.656Z{"msg":"Crucible a2dfa6ae-bb4b-47c9-895f-240a20dd1dad has session id: 0b77e4ac-274c-45c7-bd08-d3eb5b283a38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.726813302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92552024-02-03T17:26:39.656Z{"msg":"Upstairs opts: Upstairs UUID: a2dfa6ae-bb4b-47c9-895f-240a20dd1dad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.726878426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92562024-02-03T17:26:39.656Z{"msg":"Crucible stats registered with UUID: a2dfa6ae-bb4b-47c9-895f-240a20dd1dad","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.726922399Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92572024-02-03T17:26:39.656ZCrucible runtime is spawned
92582024-02-03T17:26:39.657Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727137482Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479}
92592024-02-03T17:26:39.657Z{"msg":"a2dfa6ae-bb4b-47c9-895f-240a20dd1dad active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727188933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92602024-02-03T17:26:39.657Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727262364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92612024-02-03T17:26:39.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727310987Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92622024-02-03T17:26:39.657Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727386079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92632024-02-03T17:26:39.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.72744801Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92642024-02-03T17:26:39.657Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727545937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92652024-02-03T17:26:39.657Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727673839Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92662024-02-03T17:26:39.657Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727782881Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92672024-02-03T17:26:39.657Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727887289Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92682024-02-03T17:26:39.657Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.727973838Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92692024-02-03T17:26:39.657Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.728044965Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92702024-02-03T17:26:39.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.728073411Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92712024-02-03T17:26:39.658Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.728104138Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92722024-02-03T17:26:39.658Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.729116756Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92732024-02-03T17:26:39.658Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.72918424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92742024-02-03T17:26:39.658Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.729442709Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92752024-02-03T17:26:39.658Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.729531111Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92762024-02-03T17:26:39.658Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.729926386Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92772024-02-03T17:26:39.658Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.730092384Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92782024-02-03T17:26:39.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.730742726Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92792024-02-03T17:26:39.658Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.730832592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92802024-02-03T17:26:39.658Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.73088135Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92812024-02-03T17:26:39.658Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.730947764Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92822024-02-03T17:26:39.658Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.731047482Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92832024-02-03T17:26:39.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.731072944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92842024-02-03T17:26:39.658Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.731129645Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92852024-02-03T17:26:39.658Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.731155601Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92862024-02-03T17:26:39.658Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.731721022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92872024-02-03T17:26:39.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732463293Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92882024-02-03T17:26:39.658Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732529811Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92892024-02-03T17:26:39.658Z{"msg":"[0]R flush_numbers[0..12]: [148, 145, 154, 154, 152, 152, 154, 154, 153, 153, 147, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732568205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92902024-02-03T17:26:39.658Z{"msg":"[0]R generation[0..12]: [79, 78, 85, 85, 83, 83, 85, 85, 84, 84, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732614311Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92912024-02-03T17:26:39.658Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.73264065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92922024-02-03T17:26:39.658Z{"msg":"[1]R flush_numbers[0..12]: [148, 145, 154, 154, 152, 152, 154, 154, 153, 153, 147, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732683488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92932024-02-03T17:26:39.658Z{"msg":"[1]R generation[0..12]: [79, 78, 85, 85, 83, 83, 85, 85, 84, 84, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732708021Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92942024-02-03T17:26:39.659Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.73273125Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92952024-02-03T17:26:39.659Z{"msg":"[2]R flush_numbers[0..12]: [148, 145, 154, 154, 152, 152, 154, 154, 153, 153, 147, 153]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732755983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92962024-02-03T17:26:39.659Z{"msg":"[2]R generation[0..12]: [79, 78, 85, 85, 83, 83, 85, 85, 84, 84, 78, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732778645Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
92972024-02-03T17:26:39.659Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732820509Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"Wait for a query_work_queue command to finish before sending IO
92982024-02-03T17:26:39.659Z}
92992024-02-03T17:26:39.659Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732848527Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93002024-02-03T17:26:39.659Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.73287089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93012024-02-03T17:26:39.659Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732911108Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93022024-02-03T17:26:39.659Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732934538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93032024-02-03T17:26:39.659Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.732996494Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93042024-02-03T17:26:39.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.733023935Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93052024-02-03T17:26:39.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.733070471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93062024-02-03T17:26:39.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.733106915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93072024-02-03T17:26:39.659Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.733177906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93082024-02-03T17:26:39.659Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.733200889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93092024-02-03T17:26:39.659Z{"msg":"a2dfa6ae-bb4b-47c9-895f-240a20dd1dad is now active with session: 0b77e4ac-274c-45c7-bd08-d3eb5b283a38","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.733222409Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93102024-02-03T17:26:39.659Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.733241917Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479,"session_id":"0b77e4ac-274c-45c7-bd08-d3eb5b283a38"}
93112024-02-03T17:26:39.659Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:38.733269779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479}
93122024-02-03T17:26:39.659ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93132024-02-03T17:26:39.659ZLoading write count information from file "/tmp/test_repair_verify.data"
93142024-02-03T17:26:39.659ZRun Repair workload
93152024-02-03T17:26:39.659Z01/30 Write block 403 len 20480 data: 9 8 10 9 9
93162024-02-03T17:26:39.659Z02/30 Read block 446 len 8192
93172024-02-03T17:26:39.659Z03/30 Write block 518 len 4096 data: 4
93182024-02-03T17:26:39.659Z04/30 Write block 578 len 24576 data: 8 9 10 9 9 9
93192024-02-03T17:26:39.660Z05/30 Read block 554 len 36864
93202024-02-03T17:26:39.660Z06/30 Write block 213 len 20480 data: 5 7 8 6 7
93212024-02-03T17:26:39.660Z07/30 Read block 301 len 4096
93222024-02-03T17:26:39.660Z08/30 Read block 459 len 24576
93232024-02-03T17:26:39.660Z09/30 Write block 223 len 28672 data: 10 11 12 10 13 11 7
93242024-02-03T17:26:39.660Z10/30 Write block 456 len 4096 data: 8
93252024-02-03T17:26:39.660Z11/30 Read block 126 len 8192
93262024-02-03T17:26:39.660Z12/30 Read block 50 len 12288
93272024-02-03T17:26:39.660Z13/30 Write block 155 len 20480 data: 7 6 6 5 7
93282024-02-03T17:26:39.660Z14/30 Write block 445 len 8192 data: 7 7
93292024-02-03T17:26:39.660Z15/30 Write block 262 len 36864 data: 7 6 7 7 9 7 9 12 15
93302024-02-03T17:26:39.660Z16/30 Read block 121 len 28672
93312024-02-03T17:26:39.660Z17/30 Read block 328 len 24576
93322024-02-03T17:26:39.660Z18/30 Read block 87 len 16384
93332024-02-03T17:26:39.660Z19/30 Read block 194 len 40960
93342024-02-03T17:26:39.660Z20/30 Write block 55 len 12288 data: 7 8 6
93352024-02-03T17:26:39.660Z21/30 Write block 525 len 24576 data: 8 10 11 10 10 8
93362024-02-03T17:26:39.660Z22/30 Flush
93372024-02-03T17:26:39.660Z23/30 Write block 353 len 20480 data: 6 7 8 11 11
93382024-02-03T17:26:39.660Z24/30 Read block 139 len 16384
93392024-02-03T17:26:39.660Z25/30 Write block 130 len 8192 data: 13 12
93402024-02-03T17:26:39.660Z26/30 Write block 493 len 12288 data: 9 8 6
93412024-02-03T17:26:39.660Z27/30 Write block 203 len 12288 data: 7 6 7
93422024-02-03T17:26:39.660Z28/30 Write block 5 len 36864 data: 7 7 9 9 8 12 11 10 9
93432024-02-03T17:26:39.661Z29/30 Read block 420 len 32768
93442024-02-03T17:26:39.661Z30/30 Write block 183 len 40960 data: 8 6 7 7 9 7 8 7 6 5
93452024-02-03T17:26:39.661Z----------------------------------------------------------------
93462024-02-03T17:26:39.661Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
93472024-02-03T17:26:39.661ZStates: Active Active Active
93482024-02-03T17:26:39.661ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93492024-02-03T17:26:39.661Z 1 Acked 1000 Write 5 Done Done Done false
93502024-02-03T17:26:39.661Z 2 Acked 1001 Read 2 Done Done Done false
93512024-02-03T17:26:39.661Z 3 Acked 1002 Write 1 Done Done Done false
93522024-02-03T17:26:39.661Z 4 Acked 1003 Write 6 Done Done Done false
93532024-02-03T17:26:39.661Z 5 Acked 1004 Read 9 Done Done Done false
93542024-02-03T17:26:39.667Z 6 Acked 1005 Write 5 Done Done Done false
93552024-02-03T17:26:39.667Z 7 Acked 1006 Read 1 Done Done Done false
93562024-02-03T17:26:39.667Z 8 Acked 1007 Read 6 Sent Done Done false
93572024-02-03T17:26:39.667Z 9 Acked 1008 Write 7 Sent Done Done false
93582024-02-03T17:26:39.667Z 10 Acked 1009 Write 1 Sent Done Done false
93592024-02-03T17:26:39.667Z 11 Acked 1010 Read 2 Sent Done Done false
93602024-02-03T17:26:39.667Z 12 Acked 1011 Read 3 Sent Done Done false
93612024-02-03T17:26:39.667Z 13 Acked 1012 Write 5 Sent Done Done false
93622024-02-03T17:26:39.667Z 14 Acked 1013 Write 2 Sent Done Done false
93632024-02-03T17:26:39.667Z 15 Acked 1014 Write 9 Sent Done Done false
93642024-02-03T17:26:39.667Z 16 Acked 1015 Read 7 Sent Done Done false
93652024-02-03T17:26:39.667Z 17 Acked 1016 Read 6 Sent Done Done false
93662024-02-03T17:26:39.667Z 18 Acked 1017 Read 4 Sent Done Done false
93672024-02-03T17:26:39.667Z 19 Acked 1018 Read 10 Sent Done Done false
93682024-02-03T17:26:39.667Z 20 Acked 1019 Write 3 Sent Done Done false
93692024-02-03T17:26:39.667Z 21 Acked 1020 Write 6 Sent Done Done false
93702024-02-03T17:26:39.667Z 22 Acked 1021 Flush 0 Sent Done Done false
93712024-02-03T17:26:39.668Z 23 Acked 1022 Write 5 Sent Done Done false
93722024-02-03T17:26:39.668Z 24 Acked 1023 Read 4 Sent Done Done false
93732024-02-03T17:26:39.668Z 25 Acked 1024 Write 2 Sent Done Done false
93742024-02-03T17:26:39.668Z 26 Acked 1025 Write 3 Sent Done Done false
93752024-02-03T17:26:39.668Z 27 Acked 1026 Write 3 Sent Done Done false
93762024-02-03T17:26:39.668Z 28 Acked 1027 Write 9 Sent Done Done false
93772024-02-03T17:26:39.668Z 29 Acked 1028 Read 8 Sent Done Done false
93782024-02-03T17:26:39.668Z 30 Acked 1029 Write 10 Sent Sent Sent false
93792024-02-03T17:26:39.668Z STATES DS:0 DS:1 DS:2 TOTAL
93802024-02-03T17:26:39.668Z New 0 0 0 0
93812024-02-03T17:26:39.668Z Sent 23 1 1 25
93822024-02-03T17:26:39.668Z Done 7 29 29 65
93832024-02-03T17:26:39.668Z Skipped 0 0 0 0
93842024-02-03T17:26:39.668Z Error 0 0 0 0
93852024-02-03T17:26:39.668ZLast Flush: 0 1021 1021
93862024-02-03T17:26:39.668ZDownstairs last five completed:
93872024-02-03T17:26:39.668ZUpstairs last five completed: 30 29 28 27 26
93882024-02-03T17:26:39.668Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:38.781564259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1479}
93892024-02-03T17:26:39.668ZWrote out file "/tmp/test_repair_verify.data"
93902024-02-03T17:26:39.668Z
93912024-02-03T17:26:39.668ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93922024-02-03T17:26:39.668ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93932024-02-03T17:26:39.668Z 0 000-019 79 79 79 148 148 148 F T T <---
93942024-02-03T17:26:39.668Z 1 020-039 78 78 78 145 145 145 F F F
93952024-02-03T17:26:39.669Z 2 040-059 85 86 86 154 155 155 F F F <---
93962024-02-03T17:26:39.669Z 3 060-079 85 85 85 154 154 154 F F F
93972024-02-03T17:26:39.669Z 4 080-099 83 83 83 152 152 152 F F F
93982024-02-03T17:26:39.669Z 5 100-119 83 83 83 152 152 152 F F F
93992024-02-03T17:26:39.669Z 6 120-139 85 85 85 154 154 154 F T T <---
94002024-02-03T17:26:39.669Z 7 140-159 85 86 86 154 155 155 F F F <---
94012024-02-03T17:26:39.669Z 8 160-179 84 84 84 153 153 153 F F F
94022024-02-03T17:26:39.669Z 9 180-199 84 84 84 153 153 153 F T T <---
94032024-02-03T17:26:39.669Z 10 200-219 78 86 86 147 155 155 T T T <---
94042024-02-03T17:26:39.669Z 11 220-239 84 86 86 153 155 155 F F F <---
94052024-02-03T17:26:39.669Z 12 240-259 84 84 84 153 153 153 F F F
94062024-02-03T17:26:39.669Z 13 260-279 84 86 86 153 155 155 F F F <---
94072024-02-03T17:26:39.669Z 14 280-299 84 84 84 153 153 153 F F F
94082024-02-03T17:26:39.669Z 15 300-319 84 84 84 153 153 153 F F F
94092024-02-03T17:26:39.669Z 16 320-339 80 80 80 149 149 149 F F F
94102024-02-03T17:26:39.669Z 17 340-359 84 84 84 153 153 153 F T T <---
94112024-02-03T17:26:39.669Z 18 360-379 83 83 83 152 152 152 F F F
94122024-02-03T17:26:39.669Z 19 380-399 76 76 76 143 143 143 F F F
94132024-02-03T17:26:39.669Z 20 400-419 84 86 86 153 155 155 T F F <---
94142024-02-03T17:26:39.669Z 21 420-439 84 84 84 153 153 153 F F F
94152024-02-03T17:26:39.669Z 22 440-459 82 86 86 150 155 155 F F F <---
94162024-02-03T17:26:39.669Z 23 460-479 85 85 85 154 154 154 F F F
94172024-02-03T17:26:39.669Z 24 480-499 84 84 84 153 153 153 F T T <---
94182024-02-03T17:26:39.669Z 25 500-519 84 86 86 153 155 155 T F F <---
94192024-02-03T17:26:39.669Z 26 520-539 85 86 86 154 155 155 F F F <---
94202024-02-03T17:26:39.670Z 27 540-559 77 77 77 144 144 144 F F F
94212024-02-03T17:26:39.670Z 28 560-579 80 86 86 149 155 155 T F F <---
94222024-02-03T17:26:39.670Z 29 580-599 80 86 86 149 155 155 T F F <---
94232024-02-03T17:26:39.670ZMax gen: 86, Max flush: 155
94242024-02-03T17:26:39.670ZError: Difference in extent metadata found!
94252024-02-03T17:26:39.670ZA Difference in extent metadata is expected here
94262024-02-03T17:26:39.670ZOn loop 43
94272024-02-03T17:26:39.670Z
94282024-02-03T17:26:39.673ZVerifying data now
94292024-02-03T17:26:46.032ZNo change after verify
94302024-02-03T17:26:46.032ZLoop: 43 Downstairs dump after verify (and repair):
94312024-02-03T17:26:46.055ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94322024-02-03T17:26:46.055Z 0 000-019 87 87 87 156 156 156 F F F
94332024-02-03T17:26:46.055Z 1 020-039 78 78 78 145 145 145 F F F
94342024-02-03T17:26:46.056Z 2 040-059 86 86 86 155 155 155 F F F
94352024-02-03T17:26:46.056Z 3 060-079 85 85 85 154 154 154 F F F
94362024-02-03T17:26:46.056Z 4 080-099 83 83 83 152 152 152 F F F
94372024-02-03T17:26:46.059Z 5 100-119 83 83 83 152 152 152 F F F
94382024-02-03T17:26:46.059Z 6 120-139 87 87 87 156 156 156 F F F
94392024-02-03T17:26:46.059Z 7 140-159 86 86 86 155 155 155 F F F
94402024-02-03T17:26:46.060Z 8 160-179 84 84 84 153 153 153 F F F
94412024-02-03T17:26:46.060Z 9 180-199 87 87 87 156 156 156 F F F
94422024-02-03T17:26:46.060Z 10 200-219 87 87 87 156 156 156 F F F
94432024-02-03T17:26:46.060Z 11 220-239 86 86 86 155 155 155 F F F
94442024-02-03T17:26:46.060Z 12 240-259 84 84 84 153 153 153 F F F
94452024-02-03T17:26:46.060Z 13 260-279 86 86 86 155 155 155 F F F
94462024-02-03T17:26:46.060Z 14 280-299 84 84 84 153 153 153 F F F
94472024-02-03T17:26:46.060Z 15 300-319 84 84 84 153 153 153 F F F
94482024-02-03T17:26:46.060Z 16 320-339 80 80 80 149 149 149 F F F
94492024-02-03T17:26:46.060Z 17 340-359 87 87 87 156 156 156 F F F
94502024-02-03T17:26:46.060Z 18 360-379 83 83 83 152 152 152 F F F
94512024-02-03T17:26:46.060Z 19 380-399 76 76 76 143 143 143 F F F
94522024-02-03T17:26:46.060Z 20 400-419 86 86 86 155 155 155 F F F
94532024-02-03T17:26:46.060Z 21 420-439 84 84 84 153 153 153 F F F
94542024-02-03T17:26:46.060Z 22 440-459 86 86 86 155 155 155 F F F
94552024-02-03T17:26:46.060Z 23 460-479 85 85 85 154 154 154 F F F
94562024-02-03T17:26:46.060Z 24 480-499 87 87 87 156 156 156 F F F
94572024-02-03T17:26:46.060Z 25 500-519 86 86 86 155 155 155 F F F
94582024-02-03T17:26:46.060Z 26 520-539 86 86 86 155 155 155 F F F
94592024-02-03T17:26:46.060Z 27 540-559 77 77 77 144 144 144 F F F
94602024-02-03T17:26:46.060Z 28 560-579 86 86 86 155 155 155 F F F
94612024-02-03T17:26:46.060Z 29 580-599 86 86 86 155 155 155 F F F
94622024-02-03T17:26:46.060ZMax gen: 87, Max flush: 156
94632024-02-03T17:26:46.072Z
94642024-02-03T17:26:46.072ZBegin loop 44
94652024-02-03T17:26:46.072ZDownstairs to restart: 1
94662024-02-03T17:26:46.095Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.260692423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487}
94672024-02-03T17:26:46.095Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261274842Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487}
94682024-02-03T17:26:46.099Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261323033Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487}
94692024-02-03T17:26:46.099ZCrucible runtime is spawned
94702024-02-03T17:26:46.099Z{"msg":"Crucible 5d6a6b4e-f386-4bf0-8ce5-7e0faacf6004 has session id: 9a4c6e4b-8495-4586-922c-8e1e88882fc2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261517747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94712024-02-03T17:26:46.099Z{"msg":"Upstairs opts: Upstairs UUID: 5d6a6b4e-f386-4bf0-8ce5-7e0faacf6004, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261605065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94722024-02-03T17:26:46.099Z{"msg":"Crucible stats registered with UUID: 5d6a6b4e-f386-4bf0-8ce5-7e0faacf6004","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261671707Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94732024-02-03T17:26:46.099Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261704573Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487}
94742024-02-03T17:26:46.099Z{"msg":"5d6a6b4e-f386-4bf0-8ce5-7e0faacf6004 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261758651Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94752024-02-03T17:26:46.099Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261787414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94762024-02-03T17:26:46.099Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261857603Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94772024-02-03T17:26:46.099Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261902492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94782024-02-03T17:26:46.099Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.261993412Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94792024-02-03T17:26:46.099Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.262040925Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94802024-02-03T17:26:46.100Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.262111943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94812024-02-03T17:26:46.100Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.262157646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94822024-02-03T17:26:46.100Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.262200076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94832024-02-03T17:26:46.100Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.262274825Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94842024-02-03T17:26:46.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.26235517Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94852024-02-03T17:26:46.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.262604586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94862024-02-03T17:26:46.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.262686878Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94872024-02-03T17:26:46.100Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.263852556Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94882024-02-03T17:26:46.100Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.263973834Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94892024-02-03T17:26:46.100Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.264060439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94902024-02-03T17:26:46.100Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.264194131Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94912024-02-03T17:26:46.100Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.264664849Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94922024-02-03T17:26:46.105Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.264857706Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94932024-02-03T17:26:46.105Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.26492842Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94942024-02-03T17:26:46.105Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.265018532Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94952024-02-03T17:26:46.105Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.265547255Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94962024-02-03T17:26:46.105Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.265640056Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94972024-02-03T17:26:46.105Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.265731884Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94982024-02-03T17:26:46.105Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.26595546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
94992024-02-03T17:26:46.105Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266030307Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95002024-02-03T17:26:46.105Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266056629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95012024-02-03T17:26:46.105Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266079104Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95022024-02-03T17:26:46.105Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266780622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95032024-02-03T17:26:46.105Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266852941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95042024-02-03T17:26:46.105Z{"msg":"[0]R flush_numbers[0..12]: [156, 145, 155, 154, 152, 152, 156, 155, 153, 156, 156, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266880097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95052024-02-03T17:26:46.105Z{"msg":"[0]R generation[0..12]: [87, 78, 86, 85, 83, 83, 87, 86, 84, 87, 87, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266903387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95062024-02-03T17:26:46.105Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266927588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95072024-02-03T17:26:46.105Z{"msg":"[1]R flush_numbers[0..12]: [156, 145, 155, 154, 152, 152, 156, 155, 153, 156, 156, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266950976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95082024-02-03T17:26:46.105Z{"msg":"[1]R generation[0..12]: [87, 78, 86, 85, 83, 83, 87, 86, 84, 87, 87, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266974632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95092024-02-03T17:26:46.105Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.266999165Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95102024-02-03T17:26:46.105Z{"msg":"[2]R flush_numbers[0..12]: [156, 145, 155, 154, 152, 152, 156, 155, 153, 156, 156, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267022138Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95112024-02-03T17:26:46.105Z{"msg":"[2]R generation[0..12]: [87, 78, 86, 85, 83, 83, 87, 86, 84, 87, 87, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267064854Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95122024-02-03T17:26:46.105Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267090767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95132024-02-03T17:26:46.105Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.26711276Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95142024-02-03T17:26:46.106Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267136701Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95152024-02-03T17:26:46.106Z{"msg":"Next flush: 157","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
95162024-02-03T17:26:46.106Zlevel":30,"time":"2024-02-03T17:26:45.267180921Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95172024-02-03T17:26:46.106Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267204459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95182024-02-03T17:26:46.106Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267246262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95192024-02-03T17:26:46.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267268407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95202024-02-03T17:26:46.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267309501Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95212024-02-03T17:26:46.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.26733341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95222024-02-03T17:26:46.106Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267356193Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95232024-02-03T17:26:46.106Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267400002Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95242024-02-03T17:26:46.106Z{"msg":"5d6a6b4e-f386-4bf0-8ce5-7e0faacf6004 is now active with session: 9a4c6e4b-8495-4586-922c-8e1e88882fc2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267442327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95252024-02-03T17:26:46.106Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267464257Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487,"session_id":"9a4c6e4b-8495-4586-922c-8e1e88882fc2"}
95262024-02-03T17:26:46.106Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:45.267503229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487}
95272024-02-03T17:26:46.106ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95282024-02-03T17:26:46.106ZLoading write count information from file "/tmp/test_repair_verify.data"
95292024-02-03T17:26:46.106ZRun Repair workload
95302024-02-03T17:26:46.106Z01/30 Write block 328 len 8192 data: 8 6
95312024-02-03T17:26:46.110Z02/30 Read block 9 len 16384
95322024-02-03T17:26:46.110Z03/30 Read block 118 len 32768
95332024-02-03T17:26:46.110Z04/30 Write block 390 len 8192 data: 4 5
95342024-02-03T17:26:46.110Z05/30 Read block 202 len 36864
95352024-02-03T17:26:46.113Z06/30 Read block 158 len 8192
95362024-02-03T17:26:46.113Z07/30 Read block 404 len 16384
95372024-02-03T17:26:46.113Z08/30 Read block 181 len 28672
95382024-02-03T17:26:46.116Z09/30 Write block 289 len 12288 data: 6 5 6
95392024-02-03T17:26:46.116Z10/30 Read block 46 len 16384
95402024-02-03T17:26:46.119Z11/30 Read block 159 len 32768
95412024-02-03T17:26:46.119Z12/30 Read block 470 len 12288
95422024-02-03T17:26:46.122Z13/30 Write block 505 len 12288 data: 12 13 12
95432024-02-03T17:26:46.122Z14/30 Write block 541 len 32768 data: 6 7 6 7 7 6 6 7
95442024-02-03T17:26:46.122Z15/30 Read block 446 len 40960
95452024-02-03T17:26:46.125Z16/30 Read block 22 len 20480
95462024-02-03T17:26:46.125Z17/30 Read block 427 len 40960
95472024-02-03T17:26:46.128Z18/30 Read block 267 len 36864
95482024-02-03T17:26:46.131Z19/30 Write block 231 len 28672 data: 8 8 8 9 9 9 9
95492024-02-03T17:26:46.131Z20/30 Write block 437 len 20480 data: 5 5 6 8 8
95502024-02-03T17:26:46.132Z21/30 Read block 318 len 40960
95512024-02-03T17:26:46.134Z22/30 Read block 590 len 20480
95522024-02-03T17:26:46.137Z23/30 Flush
95532024-02-03T17:26:46.140Z24/30 Flush
95542024-02-03T17:26:46.140Z25/30 Write block 1 len 12288 data: 3 4 5
95552024-02-03T17:26:46.140Z26/30 Write block 576 len 16384 data: 6 7 9 10
95562024-02-03T17:26:46.143Z27/30 Write block 137 len 12288 data: 12 14 11
95572024-02-03T17:26:46.143Z28/30 Read block 558 len 20480
95582024-02-03T17:26:46.147Z29/30 Read block 458 len 16384
95592024-02-03T17:26:46.147Z30/30 Write block 119 len 24576 data: 10 9 9 6 7 4
95602024-02-03T17:26:46.147Z----------------------------------------------------------------
95612024-02-03T17:26:46.147Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95622024-02-03T17:26:46.147ZStates: Active Active Active
95632024-02-03T17:26:46.147ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95642024-02-03T17:26:46.147Z 1 Acked 1000 Write 2 Done Done Done false
95652024-02-03T17:26:46.147Z 2 Acked 1001 Read 4 Done Done Done false
95662024-02-03T17:26:46.147Z 3 Acked 1002 Read 8 Done Sent Done false
95672024-02-03T17:26:46.147Z 4 Acked 1003 Write 2 Done Sent Done false
95682024-02-03T17:26:46.147Z 5 Acked 1004 Read 9 Done Sent Done false
95692024-02-03T17:26:46.148Z 6 Acked 1005 Read 2 Done Sent Done false
95702024-02-03T17:26:46.148Z 7 Acked 1006 Read 4 Done Sent Done false
95712024-02-03T17:26:46.148Z 8 Acked 1007 Read 7 Done Sent Done false
95722024-02-03T17:26:46.148Z 9 Acked 1008 Write 3 Done Sent Done false
95732024-02-03T17:26:46.148Z 10 Acked 1009 Read 4 Done Sent Done false
95742024-02-03T17:26:46.148Z 11 Acked 1010 Read 8 Done Sent Done false
95752024-02-03T17:26:46.148Z 12 Acked 1011 Read 3 Done Sent Done false
95762024-02-03T17:26:46.148Z 13 Acked 1012 Write 3 Done Sent Done false
95772024-02-03T17:26:46.148Z 14 Acked 1013 Write 8 Done Sent Done false
95782024-02-03T17:26:46.148Z 15 Acked 1014 Read 10 Done Sent Done false
95792024-02-03T17:26:46.148Z 16 Acked 1015 Read 5 Done Sent Done false
95802024-02-03T17:26:46.148Z 17 Acked 1016 Read 10 Done Sent Done false
95812024-02-03T17:26:46.148Z 18 Acked 1017 Read 9 Done Sent Done false
95822024-02-03T17:26:46.148Z 19 Acked 1018 Write 7 Done Sent Done false
95832024-02-03T17:26:46.148Z 20 Acked 1019 Write 5 Done Sent Done false
95842024-02-03T17:26:46.148Z 21 Acked 1020 Read 10 Done Sent Done false
95852024-02-03T17:26:46.148Z 22 Acked 1021 Read 5 Done Sent Done false
95862024-02-03T17:26:46.148Z 23 Acked 1022 Flush 0 Done Sent Done false
95872024-02-03T17:26:46.148Z 24 Acked 1023 Flush 0 Done Sent Done false
95882024-02-03T17:26:46.148Z 25 Acked 1024 Write 3 Done Sent Done false
95892024-02-03T17:26:46.148Z 26 Acked 1025 Write 4 Done Sent Done false
95902024-02-03T17:26:46.148Z 27 Acked 1026 Write 3 Done Sent Done false
95912024-02-03T17:26:46.148Z 28 Acked 1027 Read 5 Done Sent Done false
95922024-02-03T17:26:46.148Z 29 Acked 1028 Read 4 Done Sent Done false
95932024-02-03T17:26:46.149Z 30 Acked 1029 Write 6 Sent Sent Sent false
95942024-02-03T17:26:46.149Z STATES DS:0 DS:1 DS:2 TOTAL
95952024-02-03T17:26:46.149Z New 0 0 0 0
95962024-02-03T17:26:46.149Z Sent 1 28 1 30
95972024-02-03T17:26:46.149Z Done 29 2 29 60
95982024-02-03T17:26:46.149Z Skipped 0 0 0 0
95992024-02-03T17:26:46.149Z Error 0 0 0 0
96002024-02-03T17:26:46.149ZLast Flush: 1023 0 1023
96012024-02-03T17:26:46.149ZDownstairs last five completed:
96022024-02-03T17:26:46.149ZUpstairs last five completed: 30 29 28 27 26
96032024-02-03T17:26:46.152Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:45.313228941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1487}
96042024-02-03T17:26:46.152ZWrote out file "/tmp/test_repair_verify.data"
96052024-02-03T17:26:46.155Z
96062024-02-03T17:26:46.162ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96072024-02-03T17:26:46.189ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96082024-02-03T17:26:46.189Z 0 000-019 87 87 87 156 156 156 T F T <---
96092024-02-03T17:26:46.189Z 1 020-039 78 78 78 145 145 145 F F F
96102024-02-03T17:26:46.189Z 2 040-059 86 86 86 155 155 155 F F F
96112024-02-03T17:26:46.192Z 3 060-079 85 85 85 154 154 154 F F F
96122024-02-03T17:26:46.193Z 4 080-099 83 83 83 152 152 152 F F F
96132024-02-03T17:26:46.193Z 5 100-119 83 83 83 152 152 152 T F T <---
96142024-02-03T17:26:46.193Z 6 120-139 87 87 87 156 156 156 T F T <---
96152024-02-03T17:26:46.193Z 7 140-159 86 86 86 155 155 155 F F F
96162024-02-03T17:26:46.193Z 8 160-179 84 84 84 153 153 153 F F F
96172024-02-03T17:26:46.193Z 9 180-199 87 87 87 156 156 156 F F F
96182024-02-03T17:26:46.193Z 10 200-219 87 87 87 156 156 156 F F F
96192024-02-03T17:26:46.193Z 11 220-239 88 86 88 157 155 157 F F F <---
96202024-02-03T17:26:46.193Z 12 240-259 84 84 84 153 153 153 F F F
96212024-02-03T17:26:46.193Z 13 260-279 86 86 86 155 155 155 F F F
96222024-02-03T17:26:46.193Z 14 280-299 88 84 88 157 153 157 F F F <---
96232024-02-03T17:26:46.193Z 15 300-319 84 84 84 153 153 153 F F F
96242024-02-03T17:26:46.193Z 16 320-339 88 80 88 157 149 157 F T F <---
96252024-02-03T17:26:46.193Z 17 340-359 87 87 87 156 156 156 F F F
96262024-02-03T17:26:46.193Z 18 360-379 83 83 83 152 152 152 F F F
96272024-02-03T17:26:46.193Z 19 380-399 88 76 88 157 143 157 F F F <---
96282024-02-03T17:26:46.193Z 20 400-419 86 86 86 155 155 155 F F F
96292024-02-03T17:26:46.193Z 21 420-439 88 84 88 157 153 157 F F F <---
96302024-02-03T17:26:46.193Z 22 440-459 88 86 88 157 155 157 F F F <---
96312024-02-03T17:26:46.193Z 23 460-479 85 85 85 154 154 154 F F F
96322024-02-03T17:26:46.193Z 24 480-499 87 87 87 156 156 156 F F F
96332024-02-03T17:26:46.193Z 25 500-519 88 86 88 157 155 157 F F F <---
96342024-02-03T17:26:46.194Z 26 520-539 86 86 86 155 155 155 F F F
96352024-02-03T17:26:46.194Z 27 540-559 88 77 88 157 144 157 F F F <---
96362024-02-03T17:26:46.194Z 28 560-579 86 86 86 155 155 155 T F T <---
96372024-02-03T17:26:46.194Z 29 580-599 86 86 86 155 155 155 F F F
96382024-02-03T17:26:46.194ZMax gen: 88, Max flush: 157
96392024-02-03T17:26:46.194ZError: Difference in extent metadata found!
96402024-02-03T17:26:46.197ZA Difference in extent metadata is expected here
96412024-02-03T17:26:46.197ZOn loop 44
96422024-02-03T17:26:46.197Z
96432024-02-03T17:26:46.200ZVerifying data now
96442024-02-03T17:26:51.258ZNo change after verify
96452024-02-03T17:26:51.258ZLoop: 44 Downstairs dump after verify (and repair):
96462024-02-03T17:26:51.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96472024-02-03T17:26:51.286Z 0 000-019 89 89 89 158 158 158 F F F
96482024-02-03T17:26:51.289Z 1 020-039 78 78 78 145 145 145 F F F
96492024-02-03T17:26:51.289Z 2 040-059 86 86 86 155 155 155 F F F
96502024-02-03T17:26:51.289Z 3 060-079 85 85 85 154 154 154 F F F
96512024-02-03T17:26:51.289Z 4 080-099 83 83 83 152 152 152 F F F
96522024-02-03T17:26:51.289Z 5 100-119 89 89 89 158 158 158 F F F
96532024-02-03T17:26:51.289Z 6 120-139 89 89 89 158 158 158 F F F
96542024-02-03T17:26:51.289Z 7 140-159 86 86 86 155 155 155 F F F
96552024-02-03T17:26:51.289Z 8 160-179 84 84 84 153 153 153 F F F
96562024-02-03T17:26:51.289Z 9 180-199 87 87 87 156 156 156 F F F
96572024-02-03T17:26:51.289Z 10 200-219 87 87 87 156 156 156 F F F
96582024-02-03T17:26:51.289Z 11 220-239 88 88 88 157 157 157 F F F
96592024-02-03T17:26:51.289Z 12 240-259 84 84 84 153 153 153 F F F
96602024-02-03T17:26:51.289Z 13 260-279 86 86 86 155 155 155 F F F
96612024-02-03T17:26:51.289Z 14 280-299 88 88 88 157 157 157 F F F
96622024-02-03T17:26:51.289Z 15 300-319 84 84 84 153 153 153 F F F
96632024-02-03T17:26:51.289Z 16 320-339 88 88 88 157 157 157 F F F
96642024-02-03T17:26:51.290Z 17 340-359 87 87 87 156 156 156 F F F
96652024-02-03T17:26:51.290Z 18 360-379 83 83 83 152 152 152 F F F
96662024-02-03T17:26:51.290Z 19 380-399 88 88 88 157 157 157 F F F
96672024-02-03T17:26:51.290Z 20 400-419 86 86 86 155 155 155 F F F
96682024-02-03T17:26:51.290Z 21 420-439 88 88 88 157 157 157 F F F
96692024-02-03T17:26:51.290Z 22 440-459 88 88 88 157 157 157 F F F
96702024-02-03T17:26:51.290Z 23 460-479 85 85 85 154 154 154 F F F
96712024-02-03T17:26:51.290Z 24 480-499 87 87 87 156 156 156 F F F
96722024-02-03T17:26:51.290Z 25 500-519 88 88 88 157 157 157 F F F
96732024-02-03T17:26:51.290Z 26 520-539 86 86 86 155 155 155 F F F
96742024-02-03T17:26:51.290Z 27 540-559 88 88 88 157 157 157 F F F
96752024-02-03T17:26:51.290Z 28 560-579 89 89 89 158 158 158 F F F
96762024-02-03T17:26:51.290Z 29 580-599 86 86 86 155 155 155 F F F
96772024-02-03T17:26:51.290ZMax gen: 89, Max flush: 158
96782024-02-03T17:26:51.290Z
96792024-02-03T17:26:51.290ZBegin loop 45
96802024-02-03T17:26:51.290ZDownstairs to restart: 0
96812024-02-03T17:26:51.320Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.486400597Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495}
96822024-02-03T17:26:51.320Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.486979712Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495}
96832024-02-03T17:26:51.324Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487024239Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495}
96842024-02-03T17:26:51.324Z{"msg":"Crucible fc053c6d-da5c-4e87-b9b9-d11ed5023dac has session id: c6093dc4-bced-41bb-bffe-194a4e9c12dc","v":0,"name":"crucible","level":30Crucible runtime is spawned
96852024-02-03T17:26:51.325Z,"time":"2024-02-03T17:26:50.487094565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96862024-02-03T17:26:51.325Z{"msg":"Upstairs opts: Upstairs UUID: fc053c6d-da5c-4e87-b9b9-d11ed5023dac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487224489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96872024-02-03T17:26:51.325Z{"msg":"Crucible stats registered with UUID: fc053c6d-da5c-4e87-b9b9-d11ed5023dac","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487307256Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96882024-02-03T17:26:51.325Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487377378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495}
96892024-02-03T17:26:51.325Z{"msg":"fc053c6d-da5c-4e87-b9b9-d11ed5023dac active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487442181Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96902024-02-03T17:26:51.325Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.48751511Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96912024-02-03T17:26:51.325Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487557999Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96922024-02-03T17:26:51.325Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487627905Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96932024-02-03T17:26:51.325Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487671681Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96942024-02-03T17:26:51.325Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487753989Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96952024-02-03T17:26:51.325Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.487827898Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96962024-02-03T17:26:51.325Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.48807566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96972024-02-03T17:26:51.325Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.48815955Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96982024-02-03T17:26:51.325Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.488232752Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
96992024-02-03T17:26:51.325Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.488324276Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97002024-02-03T17:26:51.325Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.488368673Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97012024-02-03T17:26:51.325Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.488423601Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97022024-02-03T17:26:51.325Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.489649537Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97032024-02-03T17:26:51.325Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.489726152Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97042024-02-03T17:26:51.325Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.489885716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97052024-02-03T17:26:51.325Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.48999471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97062024-02-03T17:26:51.330Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.490610583Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97072024-02-03T17:26:51.330Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.490774911Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97082024-02-03T17:26:51.330Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.490971187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97092024-02-03T17:26:51.330Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.491039352Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97102024-02-03T17:26:51.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.491552517Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97112024-02-03T17:26:51.330Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.491639989Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97122024-02-03T17:26:51.330Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.49170961Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97132024-02-03T17:26:51.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.491759836Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97142024-02-03T17:26:51.330Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.491810999Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97152024-02-03T17:26:51.330Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.491835945Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97162024-02-03T17:26:51.330Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.492310953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97172024-02-03T17:26:51.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493036077Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97182024-02-03T17:26:51.330Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493134714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97192024-02-03T17:26:51.330Z{"msg":"[0]R flush_numbers[0..12]: [158, 145, 155, 154, 152, 158, 158, 155, 153, 156, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.49319639Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97202024-02-03T17:26:51.330Z{"msg":"[0]R generation[0..12]: [89, 78, 86, 85, 83, 89, 89, 86, 84, 87, 87, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.49322363Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97212024-02-03T17:26:51.330Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493247592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97222024-02-03T17:26:51.331Z{"msg":"[1]R flush_numbers[0..12]: [158, 145, 155, 154, 152, 158, 158, 155, 153, 156, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493272281Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97232024-02-03T17:26:51.331Z{"msg":"[1]R generation[0..12]: [89, 78, 86, 85, 83, 89, 89, 86, 84, 87, 87, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.49329518Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97242024-02-03T17:26:51.331Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493348459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97252024-02-03T17:26:51.331Z{"msg":"[2]R flush_numbers[0..12]: [158, 145, 155, 154, 152, 158, 158, 155, 153, 156, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493374543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"Wait for a query_work_queue command to finish before sending IO
97262024-02-03T17:26:51.331Z}
97272024-02-03T17:26:51.331Z{"msg":"[2]R generation[0..12]: [89, 78, 86, 85, 83, 89, 89, 86, 84, 87, 87, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493402835Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97282024-02-03T17:26:51.331Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493426474Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97292024-02-03T17:26:51.331Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493468965Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97302024-02-03T17:26:51.331Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493493856Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97312024-02-03T17:26:51.331Z{"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493516382Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97322024-02-03T17:26:51.331Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493557058Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97332024-02-03T17:26:51.331Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493581804Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97342024-02-03T17:26:51.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493601843Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97352024-02-03T17:26:51.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493648984Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97362024-02-03T17:26:51.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493676447Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97372024-02-03T17:26:51.331Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493722323Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97382024-02-03T17:26:51.331Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493744973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97392024-02-03T17:26:51.331Z{"msg":"fc053c6d-da5c-4e87-b9b9-d11ed5023dac is now active with session: c6093dc4-bced-41bb-bffe-194a4e9c12dc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493766288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97402024-02-03T17:26:51.331Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.49380623Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495,"session_id":"c6093dc4-bced-41bb-bffe-194a4e9c12dc"}
97412024-02-03T17:26:51.331Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:50.493828341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495}
97422024-02-03T17:26:51.331ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97432024-02-03T17:26:51.331ZLoading write count information from file "/tmp/test_repair_verify.data"
97442024-02-03T17:26:51.331ZRun Repair workload
97452024-02-03T17:26:51.331Z01/30 Flush
97462024-02-03T17:26:51.335Z02/30 Write block 505 len 32768 data: 13 14 13 13 13 10 7 5
97472024-02-03T17:26:51.335Z03/30 Write block 43 len 8192 data: 7 7
97482024-02-03T17:26:51.335Z04/30 Flush
97492024-02-03T17:26:51.338Z05/30 Write block 482 len 12288 data: 8 7 5
97502024-02-03T17:26:51.338Z06/30 Write block 154 len 28672 data: 8 8 7 7 6 8 8
97512024-02-03T17:26:51.338Z07/30 Read block 487 len 20480
97522024-02-03T17:26:51.341Z08/30 Write block 419 len 8192 data: 9 9
97532024-02-03T17:26:51.341Z09/30 Read block 29 len 8192
97542024-02-03T17:26:51.344Z10/30 Write block 498 len 32768 data: 9 9 10 10 10 10 10 14
97552024-02-03T17:26:51.345Z11/30 Read block 287 len 40960
97562024-02-03T17:26:51.348Z12/30 Read block 107 len 20480
97572024-02-03T17:26:51.348Z13/30 Write block 48 len 4096 data: 7
97582024-02-03T17:26:51.351Z14/30 Read block 548 len 8192
97592024-02-03T17:26:51.351Z15/30 Write block 586 len 24576 data: 9 7 7 8 9 9
97602024-02-03T17:26:51.351Z16/30 Read block 372 len 32768
97612024-02-03T17:26:51.355Z17/30 Write block 262 len 36864 data: 8 7 8 8 10 8 10 13 16
97622024-02-03T17:26:51.355Z18/30 Read block 113 len 40960
97632024-02-03T17:26:51.358Z19/30 Flush
97642024-02-03T17:26:51.361Z20/30 Write block 375 len 28672 data: 4 7 7 7 6 6 6
97652024-02-03T17:26:51.364Z21/30 Flush
97662024-02-03T17:26:51.367Z22/30 Write block 435 len 32768 data: 5 5 6 6 7 9 9 4
97672024-02-03T17:26:51.367Z23/30 Write block 593 len 4096 data: 8
97682024-02-03T17:26:51.367Z24/30 Write block 16 len 16384 data: 5 4 4 3
97692024-02-03T17:26:51.371Z25/30 Read block 496 len 36864
97702024-02-03T17:26:51.371Z26/30 Read block 396 len 8192
97712024-02-03T17:26:51.373Z27/30 Read block 166 len 12288
97722024-02-03T17:26:51.374Z28/30 Write block 92 len 12288 data: 8 8 9
97732024-02-03T17:26:51.374Z29/30 Read block 395 len 8192
97742024-02-03T17:26:51.378Z30/30 Write block 58 len 16384 data: 7 8 7 6
97752024-02-03T17:26:51.378Z----------------------------------------------------------------
97762024-02-03T17:26:51.378Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:29
97772024-02-03T17:26:51.378ZStates: Active Active Active
97782024-02-03T17:26:51.378ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97792024-02-03T17:26:51.378Z 2 Acked 1001 Write 8 Done Done Done false
97802024-02-03T17:26:51.378Z 3 Acked 1002 Write 2 Done Done Done false
97812024-02-03T17:26:51.378Z 4 Acked 1003 Flush 0 Sent Done Done false
97822024-02-03T17:26:51.378Z 5 Acked 1004 Write 3 Sent Done Done false
97832024-02-03T17:26:51.378Z 6 Acked 1005 Write 7 Sent Done Done false
97842024-02-03T17:26:51.378Z 7 Acked 1006 Read 5 Sent Done Done false
97852024-02-03T17:26:51.378Z 8 Acked 1007 Write 2 Sent Done Done false
97862024-02-03T17:26:51.378Z 9 Acked 1008 Read 2 Sent Done Done false
97872024-02-03T17:26:51.378Z 10 Acked 1009 Write 8 Sent Done Done false
97882024-02-03T17:26:51.378Z 11 Acked 1010 Read 10 Sent Done Done false
97892024-02-03T17:26:51.378Z 12 Acked 1011 Read 5 Sent Done Done false
97902024-02-03T17:26:51.378Z 13 Acked 1012 Write 1 Sent Done Done false
97912024-02-03T17:26:51.378Z 14 Acked 1013 Read 2 Sent Done Done false
97922024-02-03T17:26:51.378Z 15 Acked 1014 Write 6 Sent Done Done false
97932024-02-03T17:26:51.378Z 16 Acked 1015 Read 8 Sent Done Done false
97942024-02-03T17:26:51.378Z 17 Acked 1016 Write 9 Sent Done Done false
97952024-02-03T17:26:51.378Z 18 Acked 1017 Read 10 Sent Done Done false
97962024-02-03T17:26:51.379Z 19 Acked 1018 Flush 0 Sent Done Done false
97972024-02-03T17:26:51.379Z 20 Acked 1019 Write 7 Sent Done Done false
97982024-02-03T17:26:51.379Z 21 Acked 1020 Flush 0 Sent Done Done false
97992024-02-03T17:26:51.379Z 22 Acked 1021 Write 8 Sent Done Done false
98002024-02-03T17:26:51.379Z 23 Acked 1022 Write 1 Sent Done Done false
98012024-02-03T17:26:51.379Z 24 Acked 1023 Write 4 Sent Done Done false
98022024-02-03T17:26:51.379Z 25 Acked 1024 Read 9 Sent Done Done false
98032024-02-03T17:26:51.379Z 26 Acked 1025 Read 2 Sent Done Done false
98042024-02-03T17:26:51.379Z 27 Acked 1026 Read 3 Sent Done Done false
98052024-02-03T17:26:51.379Z 28 Acked 1027 Write 3 Sent Done Done false
98062024-02-03T17:26:51.379Z 29 Acked 1028 Read 2 Sent Done Done false
98072024-02-03T17:26:51.379Z 30 Acked 1029 Write 4 Sent Sent Sent false
98082024-02-03T17:26:51.379Z STATES DS:0 DS:1 DS:2 TOTAL
98092024-02-03T17:26:51.379Z New 0 0 0 0
98102024-02-03T17:26:51.379Z Sent 27 1 1 29
98112024-02-03T17:26:51.379Z Done 2 28 28 58
98122024-02-03T17:26:51.379Z Skipped 0 0 0 0
98132024-02-03T17:26:51.379Z Error 0 0 0 0
98142024-02-03T17:26:51.379ZLast Flush: 1000 1020 1020
98152024-02-03T17:26:51.379ZDownstairs last five completed: 1000
98162024-02-03T17:26:51.379ZUpstairs last five completed: 30 29 28 27 26
98172024-02-03T17:26:51.379Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:50.542384851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1495}
98182024-02-03T17:26:51.379ZWrote out file "/tmp/test_repair_verify.data"
98192024-02-03T17:26:51.384Z
98202024-02-03T17:26:51.391ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98212024-02-03T17:26:51.420ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98222024-02-03T17:26:51.420Z 0 000-019 89 89 89 158 158 158 F T T <---
98232024-02-03T17:26:51.420Z 1 020-039 78 78 78 145 145 145 F F F
98242024-02-03T17:26:51.420Z 2 040-059 86 90 90 155 161 161 T T T <---
98252024-02-03T17:26:51.420Z 3 060-079 85 85 85 154 154 154 F T T <---
98262024-02-03T17:26:51.420Z 4 080-099 83 83 83 152 152 152 F T T <---
98272024-02-03T17:26:51.420Z 5 100-119 89 89 89 158 158 158 F F F
98282024-02-03T17:26:51.420Z 6 120-139 89 89 89 158 158 158 F F F
98292024-02-03T17:26:51.423Z 7 140-159 86 90 90 155 161 161 F F F <---
98302024-02-03T17:26:51.423Z 8 160-179 84 90 90 153 161 161 F F F <---
98312024-02-03T17:26:51.423Z 9 180-199 87 87 87 156 156 156 F F F
98322024-02-03T17:26:51.423Z 10 200-219 87 87 87 156 156 156 F F F
98332024-02-03T17:26:51.423Z 11 220-239 88 88 88 157 157 157 F F F
98342024-02-03T17:26:51.423Z 12 240-259 84 84 84 153 153 153 F F F
98352024-02-03T17:26:51.423Z 13 260-279 86 90 90 155 161 161 F F F <---
98362024-02-03T17:26:51.424Z 14 280-299 88 88 88 157 157 157 F F F
98372024-02-03T17:26:51.424Z 15 300-319 84 84 84 153 153 153 F F F
98382024-02-03T17:26:51.424Z 16 320-339 88 88 88 157 157 157 F F F
98392024-02-03T17:26:51.424Z 17 340-359 87 87 87 156 156 156 F F F
98402024-02-03T17:26:51.424Z 18 360-379 83 90 90 152 162 162 F F F <---
98412024-02-03T17:26:51.424Z 19 380-399 88 90 90 157 162 162 F F F <---
98422024-02-03T17:26:51.424Z 20 400-419 86 90 90 155 161 161 F F F <---
98432024-02-03T17:26:51.424Z 21 420-439 88 90 90 157 161 161 F T T <---
98442024-02-03T17:26:51.424Z 22 440-459 88 88 88 157 157 157 F T T <---
98452024-02-03T17:26:51.424Z 23 460-479 85 85 85 154 154 154 F F F
98462024-02-03T17:26:51.424Z 24 480-499 87 90 90 156 161 161 F F F <---
98472024-02-03T17:26:51.424Z 25 500-519 88 90 90 157 161 161 T F F <---
98482024-02-03T17:26:51.424Z 26 520-539 86 86 86 155 155 155 F F F
98492024-02-03T17:26:51.424Z 27 540-559 88 88 88 157 157 157 F F F
98502024-02-03T17:26:51.424Z 28 560-579 89 89 89 158 158 158 F F F
98512024-02-03T17:26:51.424Z 29 580-599 86 90 90 155 161 161 F T T <---
98522024-02-03T17:26:51.424ZMax gen: 90, Max flush: 162
98532024-02-03T17:26:51.424ZError: Difference in extent metadata found!
98542024-02-03T17:26:51.431ZA Difference in extent metadata is expected here
98552024-02-03T17:26:51.431ZOn loop 45
98562024-02-03T17:26:51.431Z
98572024-02-03T17:26:51.434ZVerifying data now
98582024-02-03T17:26:56.719ZNo change after verify
98592024-02-03T17:26:56.719ZLoop: 45 Downstairs dump after verify (and repair):
98602024-02-03T17:26:56.742ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98612024-02-03T17:26:56.742Z 0 000-019 91 91 91 163 163 163 F F F
98622024-02-03T17:26:56.745Z 1 020-039 78 78 78 145 145 145 F F F
98632024-02-03T17:26:56.746Z 2 040-059 91 91 91 163 163 163 F F F
98642024-02-03T17:26:56.746Z 3 060-079 91 91 91 163 163 163 F F F
98652024-02-03T17:26:56.746Z 4 080-099 91 91 91 163 163 163 F F F
98662024-02-03T17:26:56.746Z 5 100-119 89 89 89 158 158 158 F F F
98672024-02-03T17:26:56.746Z 6 120-139 89 89 89 158 158 158 F F F
98682024-02-03T17:26:56.746Z 7 140-159 90 90 90 161 161 161 F F F
98692024-02-03T17:26:56.746Z 8 160-179 90 90 90 161 161 161 F F F
98702024-02-03T17:26:56.746Z 9 180-199 87 87 87 156 156 156 F F F
98712024-02-03T17:26:56.746Z 10 200-219 87 87 87 156 156 156 F F F
98722024-02-03T17:26:56.746Z 11 220-239 88 88 88 157 157 157 F F F
98732024-02-03T17:26:56.746Z 12 240-259 84 84 84 153 153 153 F F F
98742024-02-03T17:26:56.746Z 13 260-279 90 90 90 161 161 161 F F F
98752024-02-03T17:26:56.746Z 14 280-299 88 88 88 157 157 157 F F F
98762024-02-03T17:26:56.747Z 15 300-319 84 84 84 153 153 153 F F F
98772024-02-03T17:26:56.747Z 16 320-339 88 88 88 157 157 157 F F F
98782024-02-03T17:26:56.747Z 17 340-359 87 87 87 156 156 156 F F F
98792024-02-03T17:26:56.747Z 18 360-379 90 90 90 162 162 162 F F F
98802024-02-03T17:26:56.747Z 19 380-399 90 90 90 162 162 162 F F F
98812024-02-03T17:26:56.747Z 20 400-419 90 90 90 161 161 161 F F F
98822024-02-03T17:26:56.747Z 21 420-439 91 91 91 163 163 163 F F F
98832024-02-03T17:26:56.747Z 22 440-459 91 91 91 163 163 163 F F F
98842024-02-03T17:26:56.747Z 23 460-479 85 85 85 154 154 154 F F F
98852024-02-03T17:26:56.747Z 24 480-499 90 90 90 161 161 161 F F F
98862024-02-03T17:26:56.747Z 25 500-519 90 90 90 161 161 161 F F F
98872024-02-03T17:26:56.747Z 26 520-539 86 86 86 155 155 155 F F F
98882024-02-03T17:26:56.747Z 27 540-559 88 88 88 157 157 157 F F F
98892024-02-03T17:26:56.748Z 28 560-579 89 89 89 158 158 158 F F F
98902024-02-03T17:26:56.748Z 29 580-599 91 91 91 163 163 163 F F F
98912024-02-03T17:26:56.748ZMax gen: 91, Max flush: 163
98922024-02-03T17:26:56.751Z
98932024-02-03T17:26:56.751ZBegin loop 46
98942024-02-03T17:26:56.751ZDownstairs to restart: 1
98952024-02-03T17:26:56.779Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.945328224Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503}
98962024-02-03T17:26:56.783Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.945891769Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503}
98972024-02-03T17:26:56.784Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.945964469Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503}
98982024-02-03T17:26:56.784Z{"msg":"Crucible d33ea77b-ee13-4d62-81a8-1f5bb70842fb has session id: 1487e365-0f3b-443b-9918-c11845f89208","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.9459917Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,Crucible runtime is spawned
98992024-02-03T17:26:56.784Z"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99002024-02-03T17:26:56.784Z{"msg":"Upstairs opts: Upstairs UUID: d33ea77b-ee13-4d62-81a8-1f5bb70842fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946176218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99012024-02-03T17:26:56.784Z{"msg":"Crucible stats registered with UUID: d33ea77b-ee13-4d62-81a8-1f5bb70842fb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946242599Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99022024-02-03T17:26:56.784Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946285145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503}
99032024-02-03T17:26:56.784Z{"msg":"d33ea77b-ee13-4d62-81a8-1f5bb70842fb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.94633722Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99042024-02-03T17:26:56.784Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946378285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99052024-02-03T17:26:56.784Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.94644297Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99062024-02-03T17:26:56.784Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946485655Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99072024-02-03T17:26:56.784Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.94652624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99082024-02-03T17:26:56.784Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946569055Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99092024-02-03T17:26:56.784Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946611733Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99102024-02-03T17:26:56.785Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946673468Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99112024-02-03T17:26:56.785Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946720928Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99122024-02-03T17:26:56.785Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.94676315Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99132024-02-03T17:26:56.785Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.946983972Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99142024-02-03T17:26:56.785Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.947047307Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99152024-02-03T17:26:56.785Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.94709497Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99162024-02-03T17:26:56.785Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.948428763Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99172024-02-03T17:26:56.785Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.948586145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99182024-02-03T17:26:56.785Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.948669643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99192024-02-03T17:26:56.785Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.948718984Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99202024-02-03T17:26:56.785Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.948831274Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99212024-02-03T17:26:56.785Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.948934011Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99222024-02-03T17:26:56.785Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.949377095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99232024-02-03T17:26:56.791Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.9496726Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99242024-02-03T17:26:56.791Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.950138724Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99252024-02-03T17:26:56.791Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.950210493Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99262024-02-03T17:26:56.791Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.950237603Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99272024-02-03T17:26:56.791Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.950260411Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99282024-02-03T17:26:56.791Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.950595417Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99292024-02-03T17:26:56.791Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.950657855Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99302024-02-03T17:26:56.791Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.950702346Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99312024-02-03T17:26:56.791Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951133221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99322024-02-03T17:26:56.791Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951198215Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99332024-02-03T17:26:56.791Z{"msg":"[0]R flush_numbers[0..12]: [163, 145, 163, 163, 163, 158, 158, 161, 161, 156, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951226554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99342024-02-03T17:26:56.791Z{"msg":"[0]R generation[0..12]: [91, 78, 91, 91, 91, 89, 89, 90, 90, 87, 87, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951250697Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99352024-02-03T17:26:56.791Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951275342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99362024-02-03T17:26:56.792Z{"msg":"[1]R flush_numbers[0..12]: [163, 145, 163, 163, 163, 158, 158, 161, 161, 156, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951319441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99372024-02-03T17:26:56.792Z{"msg":"[1]R generation[0..12]: [91, 78, 91, 91, 91, 89, 89, 90, 90, 87, 87, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951344998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99382024-02-03T17:26:56.792Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951387322Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99392024-02-03T17:26:56.792Z{"msg":"[2]R flush_numbers[0..12]: [163, 145, 163, 163, 163, 158, 158, 161, 161, 156, 156, 157]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951413273Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99402024-02-03T17:26:56.792Z{"msg":"[2]R generation[0..12]: [91, 78, 91, 91, 91, 89, 89, 90, 90, 87, 87, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951454861Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"Wait for a query_work_queue command to finish before sending IO
99412024-02-03T17:26:56.792Z}
99422024-02-03T17:26:56.792Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951484989Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99432024-02-03T17:26:56.792Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.95150711Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99442024-02-03T17:26:56.792Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951529924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99452024-02-03T17:26:56.792Z{"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.95155148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99462024-02-03T17:26:56.792Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951605428Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99472024-02-03T17:26:56.792Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951667933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99482024-02-03T17:26:56.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951691797Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99492024-02-03T17:26:56.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951719983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99502024-02-03T17:26:56.792Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951784123Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99512024-02-03T17:26:56.792Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951826158Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99522024-02-03T17:26:56.792Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951876623Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99532024-02-03T17:26:56.792Z{"msg":"d33ea77b-ee13-4d62-81a8-1f5bb70842fb is now active with session: 1487e365-0f3b-443b-9918-c11845f89208","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951911407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99542024-02-03T17:26:56.793Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.951968488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503,"session_id":"1487e365-0f3b-443b-9918-c11845f89208"}
99552024-02-03T17:26:56.793Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:26:55.952006424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503}
99562024-02-03T17:26:56.793ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99572024-02-03T17:26:56.793ZLoading write count information from file "/tmp/test_repair_verify.data"
99582024-02-03T17:26:56.793ZRun Repair workload
99592024-02-03T17:26:56.793Z01/30 Write block 173 len 36864 data: 6 7 8 8 11 10 8 9 10
99602024-02-03T17:26:56.793Z02/30 Read block 378 len 4096
99612024-02-03T17:26:56.797Z03/30 Read block 246 len 4096
99622024-02-03T17:26:56.797Z04/30 Read block 464 len 8192
99632024-02-03T17:26:56.797Z05/30 Read block 507 len 12288
99642024-02-03T17:26:56.797Z06/30 Read block 47 len 32768
99652024-02-03T17:26:56.797Z07/30 Read block 574 len 40960
99662024-02-03T17:26:56.810Z08/30 Read block 63 len 20480
99672024-02-03T17:26:56.810Z09/30 Write block 545 len 32768 data: 8 7 7 8 7 7 8 7
99682024-02-03T17:26:56.813Z10/30 Write block 131 len 28672 data: 13 11 8 10 10 12 13
99692024-02-03T17:26:56.813Z11/30 Read block 490 len 36864
99702024-02-03T17:26:56.813Z12/30 Read block 382 len 28672
99712024-02-03T17:26:56.813Z13/30 Write block 229 len 20480 data: 8 6 9 9 9
99722024-02-03T17:26:56.813Z14/30 Read block 375 len 16384
99732024-02-03T17:26:56.814Z15/30 Write block 97 len 4096 data: 6
99742024-02-03T17:26:56.814Z16/30 Write block 77 len 40960 data: 6 7 6 6 6 7 10 11 8 8
99752024-02-03T17:26:56.814Z17/30 Read block 29 len 40960
99762024-02-03T17:26:56.814Z18/30 Read block 251 len 36864
99772024-02-03T17:26:56.817Z19/30 Read block 271 len 24576
99782024-02-03T17:26:56.817Z20/30 Read block 157 len 36864
99792024-02-03T17:26:56.822Z21/30 Read block 529 len 28672
99802024-02-03T17:26:56.823Z22/30 Read block 210 len 20480
99812024-02-03T17:26:56.823Z23/30 Read block 575 len 24576
99822024-02-03T17:26:56.826Z24/30 Read block 566 len 16384
99832024-02-03T17:26:56.826Z25/30 Write block 414 len 36864 data: 7 3 5 7 9 10 10 7 6
99842024-02-03T17:26:56.826Z26/30 Flush
99852024-02-03T17:26:56.830Z27/30 Write block 519 len 20480 data: 3 2 2 4 6
99862024-02-03T17:26:56.834Z28/30 Write block 582 len 4096 data: 10
99872024-02-03T17:26:56.834Z29/30 Read block 328 len 28672
99882024-02-03T17:26:56.838Z30/30 Write block 24 len 36864 data: 7 8 6 6 7 6 4 6 6
99892024-02-03T17:26:56.838Z----------------------------------------------------------------
99902024-02-03T17:26:56.838Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
99912024-02-03T17:26:56.838ZStates: Active Active Active
99922024-02-03T17:26:56.838ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99932024-02-03T17:26:56.838Z 1 Acked 1000 Write 9 Done Done Done false
99942024-02-03T17:26:56.838Z 2 Acked 1001 Read 1 Done Done Done false
99952024-02-03T17:26:56.838Z 3 Acked 1002 Read 1 Done Sent Done false
99962024-02-03T17:26:56.838Z 4 Acked 1003 Read 2 Done Sent Done false
99972024-02-03T17:26:56.838Z 5 Acked 1004 Read 3 Done Sent Done false
99982024-02-03T17:26:56.838Z 6 Acked 1005 Read 8 Done Sent Done false
99992024-02-03T17:26:56.838Z 7 Acked 1006 Read 10 Done Sent Done false
100002024-02-03T17:26:56.839Z 8 Acked 1007 Read 5 Done Sent Done false
100012024-02-03T17:26:56.839Z 9 Acked 1008 Write 8 Done Sent Done false
100022024-02-03T17:26:56.839Z 10 Acked 1009 Write 7 Done Sent Done false
100032024-02-03T17:26:56.839Z 11 Acked 1010 Read 9 Done Sent Done false
100042024-02-03T17:26:56.839Z 12 Acked 1011 Read 7 Done Sent Done false
100052024-02-03T17:26:56.839Z 13 Acked 1012 Write 5 Done Sent Done false
100062024-02-03T17:26:56.839Z 14 Acked 1013 Read 4 Done Sent Done false
100072024-02-03T17:26:56.839Z 15 Acked 1014 Write 1 Done Sent Done false
100082024-02-03T17:26:56.839Z 16 Acked 1015 Write 10 Done Sent Done false
100092024-02-03T17:26:56.839Z 17 Acked 1016 Read 10 Done Sent Done false
100102024-02-03T17:26:56.839Z 18 Acked 1017 Read 9 Done Sent Done false
100112024-02-03T17:26:56.839Z 19 Acked 1018 Read 6 Done Sent Done false
100122024-02-03T17:26:56.839Z 20 Acked 1019 Read 9 Done Sent Done false
100132024-02-03T17:26:56.840Z 21 Acked 1020 Read 7 Done Sent Done false
100142024-02-03T17:26:56.840Z 22 Acked 1021 Read 5 Done Sent Done false
100152024-02-03T17:26:56.840Z 23 Acked 1022 Read 6 Done Sent Done false
100162024-02-03T17:26:56.840Z 24 Acked 1023 Read 4 Done Sent Done false
100172024-02-03T17:26:56.840Z 25 Acked 1024 Write 9 Done Sent Done false
100182024-02-03T17:26:56.840Z 26 Acked 1025 Flush 0 Done Sent Done false
100192024-02-03T17:26:56.840Z 27 Acked 1026 Write 5 Done Sent Done false
100202024-02-03T17:26:56.840Z 28 Acked 1027 Write 1 Done Sent Done false
100212024-02-03T17:26:56.840Z 29 Acked 1028 Read 7 Done Sent Done false
100222024-02-03T17:26:56.840Z 30 Acked 1029 Write 9 Sent Sent Sent false
100232024-02-03T17:26:56.840Z STATES DS:0 DS:1 DS:2 TOTAL
100242024-02-03T17:26:56.840Z New 0 0 0 0
100252024-02-03T17:26:56.840Z Sent 1 28 1 30
100262024-02-03T17:26:56.841Z Done 29 2 29 60
100272024-02-03T17:26:56.841Z Skipped 0 0 0 0
100282024-02-03T17:26:56.841Z Error 0 0 0 0
100292024-02-03T17:26:56.841ZLast Flush: 1025 0 1025
100302024-02-03T17:26:56.841ZDownstairs last five completed:
100312024-02-03T17:26:56.841ZUpstairs last five completed: 30 29 28 27 26
100322024-02-03T17:26:56.841Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:26:56.002404643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1503}
100332024-02-03T17:26:56.841ZWrote out file "/tmp/test_repair_verify.data"
100342024-02-03T17:26:56.845Z
100352024-02-03T17:26:56.852ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100362024-02-03T17:26:56.878ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100372024-02-03T17:26:56.878Z 0 000-019 91 91 91 163 163 163 F F F
100382024-02-03T17:26:56.882Z 1 020-039 78 78 78 145 145 145 T F T <---
100392024-02-03T17:26:56.882Z 2 040-059 91 91 91 163 163 163 F F F
100402024-02-03T17:26:56.882Z 3 060-079 92 91 92 164 163 164 F F F <---
100412024-02-03T17:26:56.882Z 4 080-099 92 91 92 164 163 164 F F F <---
100422024-02-03T17:26:56.882Z 5 100-119 89 89 89 158 158 158 F F F
100432024-02-03T17:26:56.882Z 6 120-139 92 89 92 164 158 164 F F F <---
100442024-02-03T17:26:56.882Z 7 140-159 90 90 90 161 161 161 F F F
100452024-02-03T17:26:56.882Z 8 160-179 92 90 92 164 161 164 F T F <---
100462024-02-03T17:26:56.883Z 9 180-199 92 87 92 164 156 164 F T F <---
100472024-02-03T17:26:56.883Z 10 200-219 87 87 87 156 156 156 F F F
100482024-02-03T17:26:56.883Z 11 220-239 92 88 92 164 157 164 F F F <---
100492024-02-03T17:26:56.883Z 12 240-259 84 84 84 153 153 153 F F F
100502024-02-03T17:26:56.883Z 13 260-279 90 90 90 161 161 161 F F F
100512024-02-03T17:26:56.883Z 14 280-299 88 88 88 157 157 157 F F F
100522024-02-03T17:26:56.883Z 15 300-319 84 84 84 153 153 153 F F F
100532024-02-03T17:26:56.883Z 16 320-339 88 88 88 157 157 157 F F F
100542024-02-03T17:26:56.883Z 17 340-359 87 87 87 156 156 156 F F F
100552024-02-03T17:26:56.883Z 18 360-379 90 90 90 162 162 162 F F F
100562024-02-03T17:26:56.883Z 19 380-399 90 90 90 162 162 162 F F F
100572024-02-03T17:26:56.883Z 20 400-419 92 90 92 164 161 164 F F F <---
100582024-02-03T17:26:56.883Z 21 420-439 92 91 92 164 163 164 F F F <---
100592024-02-03T17:26:56.884Z 22 440-459 91 91 91 163 163 163 F F F
100602024-02-03T17:26:56.884Z 23 460-479 85 85 85 154 154 154 F F F
100612024-02-03T17:26:56.884Z 24 480-499 90 90 90 161 161 161 F F F
100622024-02-03T17:26:56.884Z 25 500-519 90 90 90 161 161 161 T F T <---
100632024-02-03T17:26:56.884Z 26 520-539 86 86 86 155 155 155 T F T <---
100642024-02-03T17:26:56.884Z 27 540-559 92 88 92 164 157 164 F F F <---
100652024-02-03T17:26:56.884Z 28 560-579 89 89 89 158 158 158 F F F
100662024-02-03T17:26:56.884Z 29 580-599 91 91 91 163 163 163 T F T <---
100672024-02-03T17:26:56.884ZMax gen: 92, Max flush: 164
100682024-02-03T17:26:56.884ZError: Difference in extent metadata found!
100692024-02-03T17:26:56.888ZA Difference in extent metadata is expected here
100702024-02-03T17:26:56.888ZOn loop 46
100712024-02-03T17:26:56.888Z
100722024-02-03T17:26:56.888ZVerifying data now
100732024-02-03T17:27:02.013ZNo change after verify
100742024-02-03T17:27:02.013ZLoop: 46 Downstairs dump after verify (and repair):
100752024-02-03T17:27:02.038ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100762024-02-03T17:27:02.038Z 0 000-019 91 91 91 163 163 163 F F F
100772024-02-03T17:27:02.038Z 1 020-039 93 93 93 165 165 165 F F F
100782024-02-03T17:27:02.038Z 2 040-059 91 91 91 163 163 163 F F F
100792024-02-03T17:27:02.038Z 3 060-079 92 92 92 164 164 164 F F F
100802024-02-03T17:27:02.038Z 4 080-099 92 92 92 164 164 164 F F F
100812024-02-03T17:27:02.038Z 5 100-119 89 89 89 158 158 158 F F F
100822024-02-03T17:27:02.038Z 6 120-139 92 92 92 164 164 164 F F F
100832024-02-03T17:27:02.038Z 7 140-159 90 90 90 161 161 161 F F F
100842024-02-03T17:27:02.038Z 8 160-179 92 92 92 164 164 164 F F F
100852024-02-03T17:27:02.038Z 9 180-199 92 92 92 164 164 164 F F F
100862024-02-03T17:27:02.039Z 10 200-219 87 87 87 156 156 156 F F F
100872024-02-03T17:27:02.039Z 11 220-239 92 92 92 164 164 164 F F F
100882024-02-03T17:27:02.039Z 12 240-259 84 84 84 153 153 153 F F F
100892024-02-03T17:27:02.042Z 13 260-279 90 90 90 161 161 161 F F F
100902024-02-03T17:27:02.042Z 14 280-299 88 88 88 157 157 157 F F F
100912024-02-03T17:27:02.042Z 15 300-319 84 84 84 153 153 153 F F F
100922024-02-03T17:27:02.042Z 16 320-339 88 88 88 157 157 157 F F F
100932024-02-03T17:27:02.042Z 17 340-359 87 87 87 156 156 156 F F F
100942024-02-03T17:27:02.042Z 18 360-379 90 90 90 162 162 162 F F F
100952024-02-03T17:27:02.043Z 19 380-399 90 90 90 162 162 162 F F F
100962024-02-03T17:27:02.043Z 20 400-419 92 92 92 164 164 164 F F F
100972024-02-03T17:27:02.043Z 21 420-439 92 92 92 164 164 164 F F F
100982024-02-03T17:27:02.043Z 22 440-459 91 91 91 163 163 163 F F F
100992024-02-03T17:27:02.043Z 23 460-479 85 85 85 154 154 154 F F F
101002024-02-03T17:27:02.043Z 24 480-499 90 90 90 161 161 161 F F F
101012024-02-03T17:27:02.043Z 25 500-519 93 93 93 165 165 165 F F F
101022024-02-03T17:27:02.043Z 26 520-539 93 93 93 165 165 165 F F F
101032024-02-03T17:27:02.043Z 27 540-559 92 92 92 164 164 164 F F F
101042024-02-03T17:27:02.043Z 28 560-579 89 89 89 158 158 158 F F F
101052024-02-03T17:27:02.043Z 29 580-599 93 93 93 165 165 165 F F F
101062024-02-03T17:27:02.043ZMax gen: 93, Max flush: 165
101072024-02-03T17:27:02.046Z
101082024-02-03T17:27:02.046ZBegin loop 47
101092024-02-03T17:27:02.046ZDownstairs to restart: 2
101102024-02-03T17:27:02.080Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.245854292Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511}
101112024-02-03T17:27:02.080Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.246409443Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511}
101122024-02-03T17:27:02.085Z{Crucible runtime is spawned
101132024-02-03T17:27:02.085Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.246693283Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511}
101142024-02-03T17:27:02.085Z{"msg":"Crucible e48ac4d9-c39f-43c7-ad22-89e26c54e177 has session id: a28865b0-7e76-49ae-ba14-b5ba89dcc5ae","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.246822025Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101152024-02-03T17:27:02.085Z{"msg":"Upstairs opts: Upstairs UUID: e48ac4d9-c39f-43c7-ad22-89e26c54e177, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.246865118Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101162024-02-03T17:27:02.086Z{"msg":"Crucible stats registered with UUID: e48ac4d9-c39f-43c7-ad22-89e26c54e177","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.246928062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101172024-02-03T17:27:02.086Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.246954538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511}
101182024-02-03T17:27:02.086Z{"msg":"e48ac4d9-c39f-43c7-ad22-89e26c54e177 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.246974814Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101192024-02-03T17:27:02.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247010829Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101202024-02-03T17:27:02.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247123122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101212024-02-03T17:27:02.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247149117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101222024-02-03T17:27:02.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247208154Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101232024-02-03T17:27:02.086Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247235354Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101242024-02-03T17:27:02.086Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247257668Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101252024-02-03T17:27:02.086Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247295598Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101262024-02-03T17:27:02.086Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247442108Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101272024-02-03T17:27:02.086Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.24754714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101282024-02-03T17:27:02.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247626527Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101292024-02-03T17:27:02.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247705724Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101302024-02-03T17:27:02.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.247749367Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101312024-02-03T17:27:02.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.248926952Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101322024-02-03T17:27:02.086Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.248994823Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101332024-02-03T17:27:02.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.249021912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101342024-02-03T17:27:02.086Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.249079214Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101352024-02-03T17:27:02.086Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.250015537Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101362024-02-03T17:27:02.086Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.250266421Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101372024-02-03T17:27:02.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.250504219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101382024-02-03T17:27:02.091Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.250593313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101392024-02-03T17:27:02.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.250823627Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101402024-02-03T17:27:02.091Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.250898268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101412024-02-03T17:27:02.091Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.250933984Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101422024-02-03T17:27:02.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.251127428Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101432024-02-03T17:27:02.091Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.251215039Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101442024-02-03T17:27:02.091Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.251298993Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101452024-02-03T17:27:02.091Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.251561187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101462024-02-03T17:27:02.091Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252272238Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101472024-02-03T17:27:02.091Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252350533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101482024-02-03T17:27:02.091Z{"msg":"[0]R flush_numbers[0..12]: [163, 165, 163, 164, 164, 158, 164, 161, 164, 164, 156, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252388979Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101492024-02-03T17:27:02.091Z{"msg":"[0]R generation[0..12]: [91, 93, 91, 92, 92, 89, 92, 90, 92, 92, 87, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252435489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101502024-02-03T17:27:02.091Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252461865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101512024-02-03T17:27:02.091Z{"msg":"[1]R flush_numbers[0..12]: [163, 165, 163, 164, 164, 158, 164, 161, 164, 164, 156, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252504677Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101522024-02-03T17:27:02.091Z{"msg":"[1]R generation[0..12]: [91, 93, 91, 92, 92, 89, 92, 90, 92, 92, 87, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252529665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101532024-02-03T17:27:02.092Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252572656Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101542024-02-03T17:27:02.092Z{"msg":"[2]R flush_numbers[0..12]: [163, 165, 163, 164, 164, 158, 164, 161, 164, 164, 156, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252598286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101552024-02-03T17:27:02.092ZWait for a query_work_queue command to finish before sending IO
101562024-02-03T17:27:02.092Z{"msg":"[2]R generation[0..12]: [91, 93, 91, 92, 92, 89, 92, 90, 92, 92, 87, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252645288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101572024-02-03T17:27:02.092Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252670777Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101582024-02-03T17:27:02.092Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252712066Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101592024-02-03T17:27:02.092Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252736837Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101602024-02-03T17:27:02.092Z{"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252758521Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101612024-02-03T17:27:02.092Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.25279965Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101622024-02-03T17:27:02.092Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252824068Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101632024-02-03T17:27:02.092Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252863854Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101642024-02-03T17:27:02.092Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252887753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101652024-02-03T17:27:02.092Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252928273Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101662024-02-03T17:27:02.092Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.252952071Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101672024-02-03T17:27:02.092Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.25299216Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101682024-02-03T17:27:02.092Z{"msg":"e48ac4d9-c39f-43c7-ad22-89e26c54e177 is now active with session: a28865b0-7e76-49ae-ba14-b5ba89dcc5ae","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.253015807Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101692024-02-03T17:27:02.092Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.253036015Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511,"session_id":"a28865b0-7e76-49ae-ba14-b5ba89dcc5ae"}
101702024-02-03T17:27:02.092Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:01.253076229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511}
101712024-02-03T17:27:02.092ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101722024-02-03T17:27:02.092ZLoading write count information from file "/tmp/test_repair_verify.data"
101732024-02-03T17:27:02.092ZRun Repair workload
101742024-02-03T17:27:02.092Z01/30 Write block 324 len 36864 data: 7 7 8 7 9 7 6 6 6
101752024-02-03T17:27:02.096Z02/30 Write block 305 len 28672 data: 9 10 7 8 8 6 5
101762024-02-03T17:27:02.096Z03/30 Read block 508 len 36864
101772024-02-03T17:27:02.096Z04/30 Read block 312 len 40960
101782024-02-03T17:27:02.099Z05/30 Flush
101792024-02-03T17:27:02.103Z06/30 Write block 22 len 28672 data: 6 6 8 9 7 7 8
101802024-02-03T17:27:02.103Z07/30 Write block 569 len 24576 data: 6 5 5 4 3 4
101812024-02-03T17:27:02.103Z08/30 Read block 277 len 4096
101822024-02-03T17:27:02.106Z09/30 Read block 237 len 32768
101832024-02-03T17:27:02.107Z10/30 Write block 311 len 40960 data: 6 3 5 6 6 6 5 4 8 6
101842024-02-03T17:27:02.107Z11/30 Read block 140 len 36864
101852024-02-03T17:27:02.110Z12/30 Flush
101862024-02-03T17:27:02.114Z13/30 Write block 155 len 28672 data: 9 8 8 7 9 9 10
101872024-02-03T17:27:02.117Z14/30 Write block 238 len 24576 data: 7 10 6 6 4 3
101882024-02-03T17:27:02.117Z15/30 Flush
101892024-02-03T17:27:02.121Z16/30 Write block 342 len 40960 data: 6 6 6 6 6 8 6 6 6 6
101902024-02-03T17:27:02.121Z17/30 Read block 13 len 32768
101912024-02-03T17:27:02.124Z18/30 Read block 316 len 4096
101922024-02-03T17:27:02.124Z19/30 Read block 157 len 4096
101932024-02-03T17:27:02.127Z20/30 Write block 543 len 32768 data: 7 8 9 8 8 9 8 8
101942024-02-03T17:27:02.130Z21/30 Write block 226 len 40960 data: 11 14 12 9 7 10 10 10 10 10
101952024-02-03T17:27:02.130Z22/30 Read block 181 len 12288
101962024-02-03T17:27:02.130Z23/30 Read block 161 len 28672
101972024-02-03T17:27:02.134Z24/30 Read block 423 len 32768
101982024-02-03T17:27:02.134Z25/30 Read block 215 len 8192
101992024-02-03T17:27:02.138Z26/30 Read block 453 len 8192
102002024-02-03T17:27:02.138Z27/30 Read block 247 len 28672
102012024-02-03T17:27:02.138Z28/30 Read block 166 len 36864
102022024-02-03T17:27:02.142Z29/30 Read block 434 len 12288
102032024-02-03T17:27:02.142Z30/30 Write block 589 len 12288 data: 9 10 10
102042024-02-03T17:27:02.142Z----------------------------------------------------------------
102052024-02-03T17:27:02.142Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
102062024-02-03T17:27:02.142ZStates: Active Active Active
102072024-02-03T17:27:02.142ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102082024-02-03T17:27:02.142Z 1 Acked 1000 Write 9 Done Done Sent false
102092024-02-03T17:27:02.142Z 2 Acked 1001 Write 7 Done Done Sent false
102102024-02-03T17:27:02.142Z 3 Acked 1002 Read 9 Done Done Sent false
102112024-02-03T17:27:02.142Z 4 Acked 1003 Read 10 Done Done Sent false
102122024-02-03T17:27:02.142Z 5 Acked 1004 Flush 0 Done Done Sent false
102132024-02-03T17:27:02.142Z 6 Acked 1005 Write 7 Done Done Sent false
102142024-02-03T17:27:02.143Z 7 Acked 1006 Write 6 Done Done Sent false
102152024-02-03T17:27:02.143Z 8 Acked 1007 Read 1 Done Done Sent false
102162024-02-03T17:27:02.143Z 9 Acked 1008 Read 8 Done Done Sent false
102172024-02-03T17:27:02.143Z 10 Acked 1009 Write 10 Done Done Sent false
102182024-02-03T17:27:02.143Z 11 Acked 1010 Read 9 Done Done Sent false
102192024-02-03T17:27:02.143Z 12 Acked 1011 Flush 0 Done Done Sent false
102202024-02-03T17:27:02.143Z 13 Acked 1012 Write 7 Done Done Sent false
102212024-02-03T17:27:02.143Z 14 Acked 1013 Write 6 Done Done Sent false
102222024-02-03T17:27:02.143Z 15 Acked 1014 Flush 0 Done Done Sent false
102232024-02-03T17:27:02.143Z 16 Acked 1015 Write 10 Done Done Sent false
102242024-02-03T17:27:02.143Z 17 Acked 1016 Read 8 Done Done Sent false
102252024-02-03T17:27:02.143Z 18 Acked 1017 Read 1 Done Done Sent false
102262024-02-03T17:27:02.143Z 19 Acked 1018 Read 1 Done Done Sent false
102272024-02-03T17:27:02.143Z 20 Acked 1019 Write 8 Done Done Sent false
102282024-02-03T17:27:02.143Z 21 Acked 1020 Write 10 Done Done Sent false
102292024-02-03T17:27:02.143Z 22 Acked 1021 Read 3 Done Done Sent false
102302024-02-03T17:27:02.143Z 23 Acked 1022 Read 7 Done Done Sent false
102312024-02-03T17:27:02.143Z 24 Acked 1023 Read 8 Done Done Sent false
102322024-02-03T17:27:02.143Z 25 Acked 1024 Read 2 Done Done Sent false
102332024-02-03T17:27:02.143Z 26 Acked 1025 Read 2 Done Done Sent false
102342024-02-03T17:27:02.143Z 27 Acked 1026 Read 7 Done Done Sent false
102352024-02-03T17:27:02.143Z 28 Acked 1027 Read 9 Done Done Sent false
102362024-02-03T17:27:02.143Z 29 Acked 1028 Read 3 Done Done Sent false
102372024-02-03T17:27:02.143Z 30 Acked 1029 Write 3 Sent Sent Sent false
102382024-02-03T17:27:02.143Z STATES DS:0 DS:1 DS:2 TOTAL
102392024-02-03T17:27:02.143Z New 0 0 0 0
102402024-02-03T17:27:02.144Z Sent 1 1 30 32
102412024-02-03T17:27:02.144Z Done 29 29 0 58
102422024-02-03T17:27:02.144Z Skipped 0 0 0 0
102432024-02-03T17:27:02.144Z Error 0 0 0 0
102442024-02-03T17:27:02.144ZLast Flush: 1014 1014 0
102452024-02-03T17:27:02.144ZDownstairs last five completed:
102462024-02-03T17:27:02.144ZUpstairs last five completed: 30 29 28 27 26
102472024-02-03T17:27:02.147Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:01.308308709Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1511}
102482024-02-03T17:27:02.148ZWrote out file "/tmp/test_repair_verify.data"
102492024-02-03T17:27:02.150Z
102502024-02-03T17:27:02.157ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102512024-02-03T17:27:02.185ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102522024-02-03T17:27:02.185Z 0 000-019 91 91 91 163 163 163 F F F
102532024-02-03T17:27:02.185Z 1 020-039 94 94 93 167 167 165 F F F <---
102542024-02-03T17:27:02.185Z 2 040-059 91 91 91 163 163 163 F F F
102552024-02-03T17:27:02.185Z 3 060-079 92 92 92 164 164 164 F F F
102562024-02-03T17:27:02.189Z 4 080-099 92 92 92 164 164 164 F F F
102572024-02-03T17:27:02.189Z 5 100-119 89 89 89 158 158 158 F F F
102582024-02-03T17:27:02.189Z 6 120-139 92 92 92 164 164 164 F F F
102592024-02-03T17:27:02.189Z 7 140-159 94 94 90 168 168 161 F F F <---
102602024-02-03T17:27:02.189Z 8 160-179 94 94 92 168 168 164 F F F <---
102612024-02-03T17:27:02.189Z 9 180-199 92 92 92 164 164 164 F F F
102622024-02-03T17:27:02.189Z 10 200-219 87 87 87 156 156 156 F F F
102632024-02-03T17:27:02.189Z 11 220-239 94 94 92 168 168 164 T T F <---
102642024-02-03T17:27:02.189Z 12 240-259 94 94 84 168 168 153 F F F <---
102652024-02-03T17:27:02.189Z 13 260-279 90 90 90 161 161 161 F F F
102662024-02-03T17:27:02.189Z 14 280-299 88 88 88 157 157 157 F F F
102672024-02-03T17:27:02.189Z 15 300-319 94 94 84 167 167 153 F F F <---
102682024-02-03T17:27:02.189Z 16 320-339 94 94 88 167 167 157 F F F <---
102692024-02-03T17:27:02.189Z 17 340-359 87 87 87 156 156 156 T T F <---
102702024-02-03T17:27:02.190Z 18 360-379 90 90 90 162 162 162 F F F
102712024-02-03T17:27:02.190Z 19 380-399 90 90 90 162 162 162 F F F
102722024-02-03T17:27:02.190Z 20 400-419 92 92 92 164 164 164 F F F
102732024-02-03T17:27:02.190Z 21 420-439 92 92 92 164 164 164 F F F
102742024-02-03T17:27:02.190Z 22 440-459 91 91 91 163 163 163 F F F
102752024-02-03T17:27:02.190Z 23 460-479 85 85 85 154 154 154 F F F
102762024-02-03T17:27:02.190Z 24 480-499 90 90 90 161 161 161 F F F
102772024-02-03T17:27:02.190Z 25 500-519 93 93 93 165 165 165 F F F
102782024-02-03T17:27:02.190Z 26 520-539 93 93 93 165 165 165 F F F
102792024-02-03T17:27:02.190Z 27 540-559 92 92 92 164 164 164 T T F <---
102802024-02-03T17:27:02.190Z 28 560-579 94 94 89 167 167 158 F F F <---
102812024-02-03T17:27:02.190Z 29 580-599 93 93 93 165 165 165 T T F <---
102822024-02-03T17:27:02.190ZMax gen: 94, Max flush: 168
102832024-02-03T17:27:02.190ZError: Difference in extent metadata found!
102842024-02-03T17:27:02.194ZA Difference in extent metadata is expected here
102852024-02-03T17:27:02.194ZOn loop 47
102862024-02-03T17:27:02.194Z
102872024-02-03T17:27:02.200ZVerifying data now
102882024-02-03T17:27:07.195ZNo change after verify
102892024-02-03T17:27:07.196ZLoop: 47 Downstairs dump after verify (and repair):
102902024-02-03T17:27:07.220ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102912024-02-03T17:27:07.220Z 0 000-019 91 91 91 163 163 163 F F F
102922024-02-03T17:27:07.223Z 1 020-039 94 94 94 167 167 167 F F F
102932024-02-03T17:27:07.223Z 2 040-059 91 91 91 163 163 163 F F F
102942024-02-03T17:27:07.224Z 3 060-079 92 92 92 164 164 164 F F F
102952024-02-03T17:27:07.224Z 4 080-099 92 92 92 164 164 164 F F F
102962024-02-03T17:27:07.224Z 5 100-119 89 89 89 158 158 158 F F F
102972024-02-03T17:27:07.224Z 6 120-139 92 92 92 164 164 164 F F F
102982024-02-03T17:27:07.224Z 7 140-159 94 94 94 168 168 168 F F F
102992024-02-03T17:27:07.224Z 8 160-179 94 94 94 168 168 168 F F F
103002024-02-03T17:27:07.224Z 9 180-199 92 92 92 164 164 164 F F F
103012024-02-03T17:27:07.224Z 10 200-219 87 87 87 156 156 156 F F F
103022024-02-03T17:27:07.224Z 11 220-239 95 95 95 169 169 169 F F F
103032024-02-03T17:27:07.224Z 12 240-259 94 94 94 168 168 168 F F F
103042024-02-03T17:27:07.224Z 13 260-279 90 90 90 161 161 161 F F F
103052024-02-03T17:27:07.224Z 14 280-299 88 88 88 157 157 157 F F F
103062024-02-03T17:27:07.224Z 15 300-319 94 94 94 167 167 167 F F F
103072024-02-03T17:27:07.225Z 16 320-339 94 94 94 167 167 167 F F F
103082024-02-03T17:27:07.225Z 17 340-359 95 95 95 169 169 169 F F F
103092024-02-03T17:27:07.225Z 18 360-379 90 90 90 162 162 162 F F F
103102024-02-03T17:27:07.225Z 19 380-399 90 90 90 162 162 162 F F F
103112024-02-03T17:27:07.225Z 20 400-419 92 92 92 164 164 164 F F F
103122024-02-03T17:27:07.225Z 21 420-439 92 92 92 164 164 164 F F F
103132024-02-03T17:27:07.225Z 22 440-459 91 91 91 163 163 163 F F F
103142024-02-03T17:27:07.225Z 23 460-479 85 85 85 154 154 154 F F F
103152024-02-03T17:27:07.225Z 24 480-499 90 90 90 161 161 161 F F F
103162024-02-03T17:27:07.225Z 25 500-519 93 93 93 165 165 165 F F F
103172024-02-03T17:27:07.225Z 26 520-539 93 93 93 165 165 165 F F F
103182024-02-03T17:27:07.225Z 27 540-559 95 95 95 169 169 169 F F F
103192024-02-03T17:27:07.225Z 28 560-579 94 94 94 167 167 167 F F F
103202024-02-03T17:27:07.226Z 29 580-599 95 95 95 169 169 169 F F F
103212024-02-03T17:27:07.226ZMax gen: 95, Max flush: 169
103222024-02-03T17:27:07.229Z
103232024-02-03T17:27:07.229ZBegin loop 48
103242024-02-03T17:27:07.229ZDownstairs to restart: 1
103252024-02-03T17:27:07.260Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.426891597Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519}
103262024-02-03T17:27:07.260Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427445264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519}
103272024-02-03T17:27:07.265Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427544712Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519}
103282024-02-03T17:27:07.265ZCrucible runtime is spawned
103292024-02-03T17:27:07.265Z{"msg":"Crucible 41ee9724-a93f-4928-a60a-241b4e84c448 has session id: 02599f40-e59a-4ad3-8187-784936900cbc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427682355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103302024-02-03T17:27:07.265Z{"msg":"Upstairs opts: Upstairs UUID: 41ee9724-a93f-4928-a60a-241b4e84c448, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427773566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103312024-02-03T17:27:07.265Z{"msg":"Crucible stats registered with UUID: 41ee9724-a93f-4928-a60a-241b4e84c448","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427803316Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103322024-02-03T17:27:07.265Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427844725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519}
103332024-02-03T17:27:07.266Z{"msg":"41ee9724-a93f-4928-a60a-241b4e84c448 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427869092Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103342024-02-03T17:27:07.266Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427892912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103352024-02-03T17:27:07.266Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427917122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103362024-02-03T17:27:07.266Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427939407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103372024-02-03T17:27:07.266Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.427985229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103382024-02-03T17:27:07.266Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.42800954Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103392024-02-03T17:27:07.266Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.428032261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103402024-02-03T17:27:07.266Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.428135997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103412024-02-03T17:27:07.266Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.428300008Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103422024-02-03T17:27:07.266Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.428440586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103432024-02-03T17:27:07.266Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.428490402Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103442024-02-03T17:27:07.266Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.428571463Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103452024-02-03T17:27:07.266Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.428608539Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103462024-02-03T17:27:07.266Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.429786492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103472024-02-03T17:27:07.267Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.429972659Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103482024-02-03T17:27:07.267Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.430021973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103492024-02-03T17:27:07.267Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.430068037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103502024-02-03T17:27:07.267Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.430106488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103512024-02-03T17:27:07.267Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.43014787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103522024-02-03T17:27:07.267Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.430748924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103532024-02-03T17:27:07.267Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.430809747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103542024-02-03T17:27:07.267Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.430910271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103552024-02-03T17:27:07.271Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.431879516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103562024-02-03T17:27:07.271Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.431986258Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103572024-02-03T17:27:07.272Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432037967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103582024-02-03T17:27:07.272Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432078267Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103592024-02-03T17:27:07.272Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432151064Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103602024-02-03T17:27:07.272Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432222101Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103612024-02-03T17:27:07.272Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.43228883Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103622024-02-03T17:27:07.272Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432359355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103632024-02-03T17:27:07.272Z{"msg":"[0]R flush_numbers[0..12]: [163, 167, 163, 164, 164, 158, 164, 168, 168, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432430188Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103642024-02-03T17:27:07.272Z{"msg":"[0]R generation[0..12]: [91, 94, 91, 92, 92, 89, 92, 94, 94, 92, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432504255Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103652024-02-03T17:27:07.272Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432554324Z","hostname":"Wait for a query_work_queue command to finish before sending IO
103662024-02-03T17:27:07.272Zip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103672024-02-03T17:27:07.272Z{"msg":"[1]R flush_numbers[0..12]: [163, 167, 163, 164, 164, 158, 164, 168, 168, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432673314Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103682024-02-03T17:27:07.272Z{"msg":"[1]R generation[0..12]: [91, 94, 91, 92, 92, 89, 92, 94, 94, 92, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432749721Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103692024-02-03T17:27:07.272Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.43282635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103702024-02-03T17:27:07.272Z{"msg":"[2]R flush_numbers[0..12]: [163, 167, 163, 164, 164, 158, 164, 168, 168, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432899572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103712024-02-03T17:27:07.272Z{"msg":"[2]R generation[0..12]: [91, 94, 91, 92, 92, 89, 92, 94, 94, 92, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.432944778Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103722024-02-03T17:27:07.272Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433018338Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103732024-02-03T17:27:07.272Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433063882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103742024-02-03T17:27:07.272Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433127592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103752024-02-03T17:27:07.272Z{"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433167033Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103762024-02-03T17:27:07.272Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433235911Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103772024-02-03T17:27:07.272Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433306921Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103782024-02-03T17:27:07.272Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433342717Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103792024-02-03T17:27:07.273Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.43341478Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103802024-02-03T17:27:07.273Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433460907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103812024-02-03T17:27:07.273Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433533179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103822024-02-03T17:27:07.273Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103832024-02-03T17:27:07.273Z,"time":"2024-02-03T17:27:06.433602447Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103842024-02-03T17:27:07.273Z{"msg":"41ee9724-a93f-4928-a60a-241b4e84c448 is now active with session: 02599f40-e59a-4ad3-8187-784936900cbc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433644512Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103852024-02-03T17:27:07.273Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433667443Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519,"session_id":"02599f40-e59a-4ad3-8187-784936900cbc"}
103862024-02-03T17:27:07.273Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:06.433687366Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519}
103872024-02-03T17:27:07.273ZLoading write count information from file "/tmp/test_repair_verify.data"
103882024-02-03T17:27:07.273ZRun Repair workload
103892024-02-03T17:27:07.273Z01/30 Flush
103902024-02-03T17:27:07.273Z02/30 Write block 348 len 16384 data: 7 7 7 7
103912024-02-03T17:27:07.278Z03/30 Read block 187 len 12288
103922024-02-03T17:27:07.278Z04/30 Write block 76 len 16384 data: 5 7 8 7
103932024-02-03T17:27:07.279Z05/30 Write block 342 len 4096 data: 7
103942024-02-03T17:27:07.279Z06/30 Write block 568 len 12288 data: 7 7 6
103952024-02-03T17:27:07.279Z07/30 Read block 163 len 20480
103962024-02-03T17:27:07.282Z08/30 Read block 377 len 4096
103972024-02-03T17:27:07.282Z09/30 Read block 463 len 36864
103982024-02-03T17:27:07.282Z10/30 Write block 353 len 32768 data: 7 8 9 12 12 12 13 12
103992024-02-03T17:27:07.282Z11/30 Read block 207 len 40960
104002024-02-03T17:27:07.289Z12/30 Read block 222 len 8192
104012024-02-03T17:27:07.289Z13/30 Flush
104022024-02-03T17:27:07.307Z14/30 Write block 262 len 40960 data: 9 8 9 9 11 9 11 14 17 16
104032024-02-03T17:27:07.307Z15/30 Write block 134 len 24576 data: 11 11 13 14 15 12
104042024-02-03T17:27:07.307Z16/30 Flush
104052024-02-03T17:27:07.307Z17/30 Flush
104062024-02-03T17:27:07.307Z18/30 Write block 543 len 36864 data: 8 9 10 9 9 10 9 9 9
104072024-02-03T17:27:07.313Z19/30 Read block 514 len 36864
104082024-02-03T17:27:07.313Z20/30 Write block 9 len 16384 data: 9 13 12 11
104092024-02-03T17:27:07.313Z21/30 Write block 378 len 40960 data: 8 7 7 7 5 5 5 5 5 5
104102024-02-03T17:27:07.313Z22/30 Write block 558 len 40960 data: 8 9 6 4 3 6 6 7 9 9
104112024-02-03T17:27:07.313Z23/30 Read block 492 len 32768
104122024-02-03T17:27:07.313Z24/30 Read block 162 len 40960
104132024-02-03T17:27:07.313Z25/30 Write block 355 len 40960 data: 10 13 13 13 14 13 13 12 14 12
104142024-02-03T17:27:07.313Z26/30 Write block 330 len 40960 data: 7 7 7 7 8 6 6 5 4 5
104152024-02-03T17:27:07.313Z27/30 Write block 452 len 8192 data: 6 6
104162024-02-03T17:27:07.319Z28/30 Write block 278 len 36864 data: 13 10 8 10 11 10 10 8 7
104172024-02-03T17:27:07.319Z29/30 Read block 281 len 12288
104182024-02-03T17:27:07.319Z30/30 Read block 214 len 28672
104192024-02-03T17:27:07.319Z----------------------------------------------------------------
104202024-02-03T17:27:07.319Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:29
104212024-02-03T17:27:07.319ZStates: Active Active Active
104222024-02-03T17:27:07.319ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104232024-02-03T17:27:07.319Z 2 Acked 1001 Write 4 Done Done Done false
104242024-02-03T17:27:07.320Z 3 Acked 1002 Read 3 Done Done Done false
104252024-02-03T17:27:07.320Z 4 Acked 1003 Write 4 Done Done Done false
104262024-02-03T17:27:07.320Z 5 Acked 1004 Write 1 Done Sent Done false
104272024-02-03T17:27:07.320Z 6 Acked 1005 Write 3 Done Sent Done false
104282024-02-03T17:27:07.320Z 7 Acked 1006 Read 5 Done Sent Done false
104292024-02-03T17:27:07.320Z 8 Acked 1007 Read 1 Done Sent Done false
104302024-02-03T17:27:07.320Z 9 Acked 1008 Read 9 Done Sent Done false
104312024-02-03T17:27:07.320Z 10 Acked 1009 Write 8 Done Sent Done false
104322024-02-03T17:27:07.320Z 11 Acked 1010 Read 10 Done Sent Done false
104332024-02-03T17:27:07.320Z 12 Acked 1011 Read 2 Done Sent Done false
104342024-02-03T17:27:07.320Z 13 Acked 1012 Flush 0 Done Sent Done false
104352024-02-03T17:27:07.320Z 14 Acked 1013 Write 10 Done Sent Done false
104362024-02-03T17:27:07.321Z 15 Acked 1014 Write 6 Done Sent Done false
104372024-02-03T17:27:07.321Z 16 Acked 1015 Flush 0 Done Sent Done false
104382024-02-03T17:27:07.321Z 17 Acked 1016 Flush 0 Done Sent Done false
104392024-02-03T17:27:07.321Z 18 Acked 1017 Write 9 Done Sent Done false
104402024-02-03T17:27:07.321Z 19 Acked 1018 Read 9 Done Sent Done false
104412024-02-03T17:27:07.321Z 20 Acked 1019 Write 4 Done Sent Done false
104422024-02-03T17:27:07.321Z 21 Acked 1020 Write 10 Done Sent Done false
104432024-02-03T17:27:07.321Z 22 Acked 1021 Write 10 Done Sent Done false
104442024-02-03T17:27:07.321Z 23 Acked 1022 Read 8 Done Sent Done false
104452024-02-03T17:27:07.321Z 24 Acked 1023 Read 10 Done Sent Done false
104462024-02-03T17:27:07.321Z 25 Acked 1024 Write 10 Done Sent Done false
104472024-02-03T17:27:07.321Z 26 Acked 1025 Write 10 Done Sent Done false
104482024-02-03T17:27:07.321Z 27 Acked 1026 Write 2 Done Sent Done false
104492024-02-03T17:27:07.322Z 28 Acked 1027 Write 9 Done Sent Done false
104502024-02-03T17:27:07.322Z 29 Acked 1028 Read 3 Done Sent Done false
104512024-02-03T17:27:07.322Z 30 Acked 1029 Read 7 Sent Sent Done false
104522024-02-03T17:27:07.322Z STATES DS:0 DS:1 DS:2 TOTAL
104532024-02-03T17:27:07.322Z New 0 0 0 0
104542024-02-03T17:27:07.322Z Sent 1 26 0 27
104552024-02-03T17:27:07.322Z Done 28 3 29 60
104562024-02-03T17:27:07.322Z Skipped 0 0 0 0
104572024-02-03T17:27:07.322Z Error 0 0 0 0
104582024-02-03T17:27:07.322ZLast Flush: 1016 1000 1016
104592024-02-03T17:27:07.322ZDownstairs last five completed: 1000
104602024-02-03T17:27:07.322ZUpstairs last five completed: 30 29 28 27 26
104612024-02-03T17:27:07.323Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:06.483599095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1519}
104622024-02-03T17:27:07.323ZWrote out file "/tmp/test_repair_verify.data"
104632024-02-03T17:27:07.326Z
104642024-02-03T17:27:07.333ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104652024-02-03T17:27:07.362ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104662024-02-03T17:27:07.362Z 0 000-019 91 91 91 163 163 163 T F T <---
104672024-02-03T17:27:07.367Z 1 020-039 94 94 94 167 167 167 F F F
104682024-02-03T17:27:07.367Z 2 040-059 91 91 91 163 163 163 F F F
104692024-02-03T17:27:07.367Z 3 060-079 96 92 96 171 164 171 F T F <---
104702024-02-03T17:27:07.368Z 4 080-099 92 92 92 164 164 164 F F F
104712024-02-03T17:27:07.368Z 5 100-119 89 89 89 158 158 158 F F F
104722024-02-03T17:27:07.368Z 6 120-139 96 92 96 172 164 172 F F F <---
104732024-02-03T17:27:07.368Z 7 140-159 94 94 94 168 168 168 F F F
104742024-02-03T17:27:07.368Z 8 160-179 94 94 94 168 168 168 F F F
104752024-02-03T17:27:07.368Z 9 180-199 92 92 92 164 164 164 F F F
104762024-02-03T17:27:07.368Z 10 200-219 87 87 87 156 156 156 F F F
104772024-02-03T17:27:07.368Z 11 220-239 95 95 95 169 169 169 F F F
104782024-02-03T17:27:07.368Z 12 240-259 94 94 94 168 168 168 F F F
104792024-02-03T17:27:07.368Z 13 260-279 96 90 96 172 161 172 T F T <---
104802024-02-03T17:27:07.368Z 14 280-299 88 88 88 157 157 157 T F T <---
104812024-02-03T17:27:07.368Z 15 300-319 94 94 94 167 167 167 F F F
104822024-02-03T17:27:07.368Z 16 320-339 94 94 94 167 167 167 T F T <---
104832024-02-03T17:27:07.369Z 17 340-359 96 95 96 171 169 171 T T T <---
104842024-02-03T17:27:07.369Z 18 360-379 96 90 96 171 162 171 T F T <---
104852024-02-03T17:27:07.369Z 19 380-399 90 90 90 162 162 162 T F T <---
104862024-02-03T17:27:07.369Z 20 400-419 92 92 92 164 164 164 F F F
104872024-02-03T17:27:07.369Z 21 420-439 92 92 92 164 164 164 F F F
104882024-02-03T17:27:07.369Z 22 440-459 91 91 91 163 163 163 T F T <---
104892024-02-03T17:27:07.369Z 23 460-479 85 85 85 154 154 154 F F F
104902024-02-03T17:27:07.369Z 24 480-499 90 90 90 161 161 161 F F F
104912024-02-03T17:27:07.369Z 25 500-519 93 93 93 165 165 165 F F F
104922024-02-03T17:27:07.369Z 26 520-539 93 93 93 165 165 165 F F F
104932024-02-03T17:27:07.369Z 27 540-559 95 95 95 169 169 169 T F T <---
104942024-02-03T17:27:07.369Z 28 560-579 96 94 96 171 167 171 T F T <---
104952024-02-03T17:27:07.369Z 29 580-599 95 95 95 169 169 169 F F F
104962024-02-03T17:27:07.369ZMax gen: 96, Max flush: 172
104972024-02-03T17:27:07.370ZError: Difference in extent metadata found!
104982024-02-03T17:27:07.373ZA Difference in extent metadata is expected here
104992024-02-03T17:27:07.373ZOn loop 48
105002024-02-03T17:27:07.373Z
105012024-02-03T17:27:07.373ZVerifying data now
105022024-02-03T17:27:13.453ZNo change after verify
105032024-02-03T17:27:13.453ZLoop: 48 Downstairs dump after verify (and repair):
105042024-02-03T17:27:13.478ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105052024-02-03T17:27:13.478Z 0 000-019 97 97 97 173 173 173 F F F
105062024-02-03T17:27:13.481Z 1 020-039 94 94 94 167 167 167 F F F
105072024-02-03T17:27:13.481Z 2 040-059 91 91 91 163 163 163 F F F
105082024-02-03T17:27:13.481Z 3 060-079 96 96 96 171 171 171 F F F
105092024-02-03T17:27:13.481Z 4 080-099 92 92 92 164 164 164 F F F
105102024-02-03T17:27:13.481Z 5 100-119 89 89 89 158 158 158 F F F
105112024-02-03T17:27:13.482Z 6 120-139 96 96 96 172 172 172 F F F
105122024-02-03T17:27:13.482Z 7 140-159 94 94 94 168 168 168 F F F
105132024-02-03T17:27:13.482Z 8 160-179 94 94 94 168 168 168 F F F
105142024-02-03T17:27:13.482Z 9 180-199 92 92 92 164 164 164 F F F
105152024-02-03T17:27:13.482Z 10 200-219 87 87 87 156 156 156 F F F
105162024-02-03T17:27:13.482Z 11 220-239 95 95 95 169 169 169 F F F
105172024-02-03T17:27:13.482Z 12 240-259 94 94 94 168 168 168 F F F
105182024-02-03T17:27:13.482Z 13 260-279 97 97 97 173 173 173 F F F
105192024-02-03T17:27:13.482Z 14 280-299 97 97 97 173 173 173 F F F
105202024-02-03T17:27:13.482Z 15 300-319 94 94 94 167 167 167 F F F
105212024-02-03T17:27:13.482Z 16 320-339 97 97 97 173 173 173 F F F
105222024-02-03T17:27:13.482Z 17 340-359 97 97 97 173 173 173 F F F
105232024-02-03T17:27:13.482Z 18 360-379 97 97 97 173 173 173 F F F
105242024-02-03T17:27:13.482Z 19 380-399 97 97 97 173 173 173 F F F
105252024-02-03T17:27:13.482Z 20 400-419 92 92 92 164 164 164 F F F
105262024-02-03T17:27:13.482Z 21 420-439 92 92 92 164 164 164 F F F
105272024-02-03T17:27:13.482Z 22 440-459 97 97 97 173 173 173 F F F
105282024-02-03T17:27:13.482Z 23 460-479 85 85 85 154 154 154 F F F
105292024-02-03T17:27:13.482Z 24 480-499 90 90 90 161 161 161 F F F
105302024-02-03T17:27:13.482Z 25 500-519 93 93 93 165 165 165 F F F
105312024-02-03T17:27:13.482Z 26 520-539 93 93 93 165 165 165 F F F
105322024-02-03T17:27:13.482Z 27 540-559 97 97 97 173 173 173 F F F
105332024-02-03T17:27:13.482Z 28 560-579 97 97 97 173 173 173 F F F
105342024-02-03T17:27:13.482Z 29 580-599 95 95 95 169 169 169 F F F
105352024-02-03T17:27:13.482ZMax gen: 97, Max flush: 173
105362024-02-03T17:27:13.486Z
105372024-02-03T17:27:13.486ZBegin loop 49
105382024-02-03T17:27:13.486ZDownstairs to restart: 0
105392024-02-03T17:27:13.520Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.686617375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527}
105402024-02-03T17:27:13.520Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687165263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527}
105412024-02-03T17:27:13.520Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687194352Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527}
105422024-02-03T17:27:13.520Z{"msg":"Crucible 7d8c426c-d8d0-43b2-940d-e8a6dedfb858 has session id: 292d7305-1495-4d9c-89d0-6ca1ba13091f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687224413Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105432024-02-03T17:27:13.524Z{"msg":"Upstairs opts: Upstairs UUID: 7d8c426c-d8d0-43b2-940d-e8a6dedfb858, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687306741Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105442024-02-03T17:27:13.524Z{"msg":"Crucible stats registered with UUID: 7d8c426c-d8d0-43b2-940d-e8a6dedfb858","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687352772Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105452024-02-03T17:27:13.524ZCrucible runtime is spawned
105462024-02-03T17:27:13.524Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687651387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527}
105472024-02-03T17:27:13.524Z{"msg":"7d8c426c-d8d0-43b2-940d-e8a6dedfb858 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687790619Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105482024-02-03T17:27:13.524Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687846887Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105492024-02-03T17:27:13.524Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687873257Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105502024-02-03T17:27:13.524Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.6878959Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105512024-02-03T17:27:13.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687953413Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105522024-02-03T17:27:13.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.687996251Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105532024-02-03T17:27:13.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.688036002Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105542024-02-03T17:27:13.525Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.688075013Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105552024-02-03T17:27:13.525Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.68811779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105562024-02-03T17:27:13.525Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.688156886Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105572024-02-03T17:27:13.525Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.688539577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105582024-02-03T17:27:13.525Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.688632661Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105592024-02-03T17:27:13.525Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.688680762Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105602024-02-03T17:27:13.525Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.690091305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105612024-02-03T17:27:13.525Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.690156595Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105622024-02-03T17:27:13.525Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.690184431Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105632024-02-03T17:27:13.526Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.69020737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105642024-02-03T17:27:13.530Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.691084389Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105652024-02-03T17:27:13.530Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.691144293Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105662024-02-03T17:27:13.530Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.69117053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105672024-02-03T17:27:13.531Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.691211093Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105682024-02-03T17:27:13.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.69201151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105692024-02-03T17:27:13.531Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.692107723Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105702024-02-03T17:27:13.531Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.692187045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105712024-02-03T17:27:13.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.692229113Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105722024-02-03T17:27:13.531Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.692291265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105732024-02-03T17:27:13.531Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.69231807Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105742024-02-03T17:27:13.531Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.692488141Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105752024-02-03T17:27:13.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693258515Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105762024-02-03T17:27:13.531Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693341022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105772024-02-03T17:27:13.531Z{"msg":"[0]R flush_numbers[0..12]: [173, 167, 163, 171, 164, 158, 172, 168, 168, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693411582Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105782024-02-03T17:27:13.531Z{"msg":"[0]R generation[0..12]: [97, 94, 91, 96, 92, 89, 96, 94, 94, 92, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693473789Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105792024-02-03T17:27:13.531Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693536037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105802024-02-03T17:27:13.532Z{"msg":"[1]R flush_numbers[0..12]: [173, 167, 163, 171, 164, 158, 172, 168, 168, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
105812024-02-03T17:27:13.532Z2024-02-03T17:27:12.693600299Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105822024-02-03T17:27:13.532Z{"msg":"[1]R generation[0..12]: [97, 94, 91, 96, 92, 89, 96, 94, 94, 92, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693666067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105832024-02-03T17:27:13.532Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693698549Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105842024-02-03T17:27:13.532Z{"msg":"[2]R flush_numbers[0..12]: [173, 167, 163, 171, 164, 158, 172, 168, 168, 164, 156, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693758182Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105852024-02-03T17:27:13.532Z{"msg":"[2]R generation[0..12]: [97, 94, 91, 96, 92, 89, 96, 94, 94, 92, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693788394Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105862024-02-03T17:27:13.532Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693851774Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105872024-02-03T17:27:13.532Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693883522Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105882024-02-03T17:27:13.532Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693941713Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105892024-02-03T17:27:13.532Z{"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.693970037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105902024-02-03T17:27:13.532Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694028734Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105912024-02-03T17:27:13.532Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694125722Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105922024-02-03T17:27:13.532Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694151882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105932024-02-03T17:27:13.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694211252Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105942024-02-03T17:27:13.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694243Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105952024-02-03T17:27:13.533Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694301282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105962024-02-03T17:27:13.533Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694326028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105972024-02-03T17:27:13.533Z{"msg":"7d8c426c-d8d0-43b2-940d-e8a6dedfb858 is now active with session: 292d7305-1495-4d9c-89d0-6ca1ba13091f","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105982024-02-03T17:27:13.533Z,"time":"2024-02-03T17:27:12.694401441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
105992024-02-03T17:27:13.533Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694497072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527,"session_id":"292d7305-1495-4d9c-89d0-6ca1ba13091f"}
106002024-02-03T17:27:13.533Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:12.694552233Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527}
106012024-02-03T17:27:13.533ZLoading write count information from file "/tmp/test_repair_verify.data"
106022024-02-03T17:27:13.533ZRun Repair workload
106032024-02-03T17:27:13.533Z01/30 Write block 100 len 16384 data: 6 6 6 7
106042024-02-03T17:27:13.537Z02/30 Write block 66 len 8192 data: 6 5
106052024-02-03T17:27:13.537Z03/30 Write block 463 len 20480 data: 9 9 11 8 10
106062024-02-03T17:27:13.537Z04/30 Write block 188 len 20480 data: 8 9 8 7 6
106072024-02-03T17:27:13.538Z05/30 Flush
106082024-02-03T17:27:13.541Z06/30 Write block 503 len 28672 data: 11 11 15 15 14 14 14
106092024-02-03T17:27:13.541Z07/30 Read block 154 len 16384
106102024-02-03T17:27:13.541Z08/30 Read block 535 len 4096
106112024-02-03T17:27:13.556Z09/30 Read block 303 len 40960
106122024-02-03T17:27:13.556Z10/30 Flush
106132024-02-03T17:27:13.570Z11/30 Write block 441 len 20480 data: 10 5 6 6 8
106142024-02-03T17:27:13.570Z12/30 Read block 434 len 24576
106152024-02-03T17:27:13.570Z13/30 Read block 431 len 24576
106162024-02-03T17:27:13.570Z14/30 Read block 371 len 4096
106172024-02-03T17:27:13.570Z15/30 Read block 387 len 8192
106182024-02-03T17:27:13.570Z16/30 Write block 293 len 20480 data: 9 9 7 7 8
106192024-02-03T17:27:13.570Z17/30 Write block 83 len 16384 data: 11 12 9 9
106202024-02-03T17:27:13.570Z18/30 Write block 507 len 28672 data: 15 15 15 11 8 6 4
106212024-02-03T17:27:13.570Z19/30 Read block 569 len 4096
106222024-02-03T17:27:13.571Z20/30 Flush
106232024-02-03T17:27:13.571Z21/30 Write block 127 len 36864 data: 9 11 13 14 14 12 9 12 12
106242024-02-03T17:27:13.571Z22/30 Read block 331 len 16384
106252024-02-03T17:27:13.571Z23/30 Write block 191 len 8192 data: 8 7
106262024-02-03T17:27:13.571Z24/30 Write block 110 len 4096 data: 11
106272024-02-03T17:27:13.571Z25/30 Read block 304 len 20480
106282024-02-03T17:27:13.571Z26/30 Flush
106292024-02-03T17:27:13.580Z27/30 Write block 434 len 12288 data: 5 6 6
106302024-02-03T17:27:13.580Z28/30 Write block 536 len 4096 data: 8
106312024-02-03T17:27:13.580Z29/30 Write block 146 len 24576 data: 3 3 4 5 9 9
106322024-02-03T17:27:13.580Z30/30 Read block 462 len 32768
106332024-02-03T17:27:13.580Z----------------------------------------------------------------
106342024-02-03T17:27:13.580Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:25
106352024-02-03T17:27:13.580ZStates: Active Active Active
106362024-02-03T17:27:13.581ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106372024-02-03T17:27:13.581Z 6 Acked 1005 Write 7 Sent Done Done false
106382024-02-03T17:27:13.581Z 7 Acked 1006 Read 4 Sent Done Done false
106392024-02-03T17:27:13.581Z 8 Acked 1007 Read 1 Sent Done Done false
106402024-02-03T17:27:13.581Z 9 Acked 1008 Read 10 Sent Done Done false
106412024-02-03T17:27:13.581Z 10 Acked 1009 Flush 0 Sent Done Done false
106422024-02-03T17:27:13.581Z 11 Acked 1010 Write 5 Sent Done Done false
106432024-02-03T17:27:13.581Z 12 Acked 1011 Read 6 Sent Done Done false
106442024-02-03T17:27:13.582Z 13 Acked 1012 Read 6 Sent Done Done false
106452024-02-03T17:27:13.582Z 14 Acked 1013 Read 1 Sent Done Done false
106462024-02-03T17:27:13.582Z 15 Acked 1014 Read 2 Sent Done Done false
106472024-02-03T17:27:13.582Z 16 Acked 1015 Write 5 Sent Done Done false
106482024-02-03T17:27:13.582Z 17 Acked 1016 Write 4 Sent Done Done false
106492024-02-03T17:27:13.582Z 18 Acked 1017 Write 7 Sent Done Done false
106502024-02-03T17:27:13.582Z 19 Acked 1018 Read 1 Sent Done Done false
106512024-02-03T17:27:13.582Z 20 Acked 1019 Flush 0 Sent Done Done false
106522024-02-03T17:27:13.582Z 21 Acked 1020 Write 9 Sent Done Done false
106532024-02-03T17:27:13.582Z 22 Acked 1021 Read 4 Sent Done Done false
106542024-02-03T17:27:13.582Z 23 Acked 1022 Write 2 Sent Done Done false
106552024-02-03T17:27:13.582Z 24 Acked 1023 Write 1 Sent Done Done false
106562024-02-03T17:27:13.583Z 25 Acked 1024 Read 5 Sent Done Done false
106572024-02-03T17:27:13.583Z 26 Acked 1025 Flush 0 Sent Done Done false
106582024-02-03T17:27:13.583Z 27 Acked 1026 Write 3 Sent Done Done false
106592024-02-03T17:27:13.583Z 28 Acked 1027 Write 1 Sent Done Done false
106602024-02-03T17:27:13.583Z 29 Acked 1028 Write 6 Sent Done Done false
106612024-02-03T17:27:13.583Z 30 Acked 1029 Read 8 Sent Sent Done false
106622024-02-03T17:27:13.583Z STATES DS:0 DS:1 DS:2 TOTAL
106632024-02-03T17:27:13.583Z New 0 0 0 0
106642024-02-03T17:27:13.583Z Sent 25 1 0 26
106652024-02-03T17:27:13.584Z Done 0 24 25 49
106662024-02-03T17:27:13.584Z Skipped 0 0 0 0
106672024-02-03T17:27:13.584Z Error 0 0 0 0
106682024-02-03T17:27:13.584ZLast Flush: 1004 1025 1025
106692024-02-03T17:27:13.584ZDownstairs last five completed: 1004 1003 1002 1001 1000
106702024-02-03T17:27:13.584ZUpstairs last five completed: 30 29 28 27 26
106712024-02-03T17:27:13.584Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:12.741520787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1527}
106722024-02-03T17:27:13.584ZWrote out file "/tmp/test_repair_verify.data"
106732024-02-03T17:27:13.589Z
106742024-02-03T17:27:13.596ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106752024-02-03T17:27:13.616ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106762024-02-03T17:27:13.616Z 0 000-019 97 97 97 173 173 173 F F F
106772024-02-03T17:27:13.619Z 1 020-039 94 94 94 167 167 167 F F F
106782024-02-03T17:27:13.619Z 2 040-059 91 91 91 163 163 163 F F F
106792024-02-03T17:27:13.619Z 3 060-079 98 98 98 174 174 174 F F F
106802024-02-03T17:27:13.619Z 4 080-099 92 98 98 164 176 176 F F F <---
106812024-02-03T17:27:13.620Z 5 100-119 98 98 98 174 177 177 F F F <---
106822024-02-03T17:27:13.620Z 6 120-139 96 98 98 172 177 177 F F F <---
106832024-02-03T17:27:13.620Z 7 140-159 94 94 94 168 168 168 F T T <---
106842024-02-03T17:27:13.620Z 8 160-179 94 94 94 168 168 168 F F F
106852024-02-03T17:27:13.620Z 9 180-199 98 98 98 174 177 177 F F F <---
106862024-02-03T17:27:13.620Z 10 200-219 87 87 87 156 156 156 F F F
106872024-02-03T17:27:13.620Z 11 220-239 95 95 95 169 169 169 F F F
106882024-02-03T17:27:13.620Z 12 240-259 94 94 94 168 168 168 F F F
106892024-02-03T17:27:13.620Z 13 260-279 97 97 97 173 173 173 F F F
106902024-02-03T17:27:13.620Z 14 280-299 97 98 98 173 176 176 F F F <---
106912024-02-03T17:27:13.620Z 15 300-319 94 94 94 167 167 167 F F F
106922024-02-03T17:27:13.620Z 16 320-339 97 97 97 173 173 173 F F F
106932024-02-03T17:27:13.620Z 17 340-359 97 97 97 173 173 173 F F F
106942024-02-03T17:27:13.620Z 18 360-379 97 97 97 173 173 173 F F F
106952024-02-03T17:27:13.620Z 19 380-399 97 97 97 173 173 173 F F F
106962024-02-03T17:27:13.621Z 20 400-419 92 92 92 164 164 164 F F F
106972024-02-03T17:27:13.621Z 21 420-439 92 92 92 164 164 164 F T T <---
106982024-02-03T17:27:13.621Z 22 440-459 97 98 98 173 176 176 F F F <---
106992024-02-03T17:27:13.621Z 23 460-479 98 98 98 174 174 174 F F F
107002024-02-03T17:27:13.621Z 24 480-499 90 90 90 161 161 161 F F F
107012024-02-03T17:27:13.621Z 25 500-519 93 98 98 165 176 176 F F F <---
107022024-02-03T17:27:13.621Z 26 520-539 93 93 93 165 165 165 F T T <---
107032024-02-03T17:27:13.621Z 27 540-559 97 97 97 173 173 173 F F F
107042024-02-03T17:27:13.621Z 28 560-579 97 97 97 173 173 173 F F F
107052024-02-03T17:27:13.621Z 29 580-599 95 95 95 169 169 169 F F F
107062024-02-03T17:27:13.621ZMax gen: 98, Max flush: 177
107072024-02-03T17:27:13.621ZError: Difference in extent metadata found!
107082024-02-03T17:27:13.624ZA Difference in extent metadata is expected here
107092024-02-03T17:27:13.625ZOn loop 49
107102024-02-03T17:27:13.625Z
107112024-02-03T17:27:13.625ZVerifying data now
107122024-02-03T17:27:18.529ZNo change after verify
107132024-02-03T17:27:18.529ZLoop: 49 Downstairs dump after verify (and repair):
107142024-02-03T17:27:18.576ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107152024-02-03T17:27:18.576Z 0 000-019 97 97 97 173 173 173 F F F
107162024-02-03T17:27:18.576Z 1 020-039 94 94 94 167 167 167 F F F
107172024-02-03T17:27:18.576Z 2 040-059 91 91 91 163 163 163 F F F
107182024-02-03T17:27:18.576Z 3 060-079 98 98 98 174 174 174 F F F
107192024-02-03T17:27:18.576Z 4 080-099 98 98 98 176 176 176 F F F
107202024-02-03T17:27:18.576Z 5 100-119 98 98 98 177 177 177 F F F
107212024-02-03T17:27:18.576Z 6 120-139 98 98 98 177 177 177 F F F
107222024-02-03T17:27:18.576Z 7 140-159 99 99 99 178 178 178 F F F
107232024-02-03T17:27:18.576Z 8 160-179 94 94 94 168 168 168 F F F
107242024-02-03T17:27:18.580Z 9 180-199 98 98 98 177 177 177 F F F
107252024-02-03T17:27:18.580Z 10 200-219 87 87 87 156 156 156 F F F
107262024-02-03T17:27:18.580Z 11 220-239 95 95 95 169 169 169 F F F
107272024-02-03T17:27:18.580Z 12 240-259 94 94 94 168 168 168 F F F
107282024-02-03T17:27:18.580Z 13 260-279 97 97 97 173 173 173 F F F
107292024-02-03T17:27:18.580Z 14 280-299 98 98 98 176 176 176 F F F
107302024-02-03T17:27:18.580Z 15 300-319 94 94 94 167 167 167 F F F
107312024-02-03T17:27:18.580Z 16 320-339 97 97 97 173 173 173 F F F
107322024-02-03T17:27:18.580Z 17 340-359 97 97 97 173 173 173 F F F
107332024-02-03T17:27:18.580Z 18 360-379 97 97 97 173 173 173 F F F
107342024-02-03T17:27:18.581Z 19 380-399 97 97 97 173 173 173 F F F
107352024-02-03T17:27:18.581Z 20 400-419 92 92 92 164 164 164 F F F
107362024-02-03T17:27:18.581Z 21 420-439 99 99 99 178 178 178 F F F
107372024-02-03T17:27:18.581Z 22 440-459 98 98 98 176 176 176 F F F
107382024-02-03T17:27:18.581Z 23 460-479 98 98 98 174 174 174 F F F
107392024-02-03T17:27:18.581Z 24 480-499 90 90 90 161 161 161 F F F
107402024-02-03T17:27:18.581Z 25 500-519 98 98 98 176 176 176 F F F
107412024-02-03T17:27:18.581Z 26 520-539 99 99 99 178 178 178 F F F
107422024-02-03T17:27:18.581Z 27 540-559 97 97 97 173 173 173 F F F
107432024-02-03T17:27:18.581Z 28 560-579 97 97 97 173 173 173 F F F
107442024-02-03T17:27:18.581Z 29 580-599 95 95 95 169 169 169 F F F
107452024-02-03T17:27:18.581ZMax gen: 99, Max flush: 178
107462024-02-03T17:27:18.581Z
107472024-02-03T17:27:18.581ZBegin loop 50
107482024-02-03T17:27:18.581ZDownstairs to restart: 2
107492024-02-03T17:27:18.590Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.756785122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535}
107502024-02-03T17:27:18.590Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.757304279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535}
107512024-02-03T17:27:18.590Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.757334509Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535}
107522024-02-03T17:27:18.594Z{"msg":"Crucible 4ace36dd-526c-44e9-be7f-5ea21bf80fe1 has session id: 6f15c59e-c01f-4db9-bb5a-d927b131fdd1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.757461561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107532024-02-03T17:27:18.594Z{"msg":"Upstairs opts: Upstairs UUID: 4ace36dd-526c-44e9-be7f-5ea21bf80fe1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.757542596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","Crucible runtime is spawned
107542024-02-03T17:27:18.594Zpid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107552024-02-03T17:27:18.594Z{"msg":"Crucible stats registered with UUID: 4ace36dd-526c-44e9-be7f-5ea21bf80fe1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.757649379Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107562024-02-03T17:27:18.594Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.757731935Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535}
107572024-02-03T17:27:18.594Z{"msg":"4ace36dd-526c-44e9-be7f-5ea21bf80fe1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.757877624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107582024-02-03T17:27:18.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.757953179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107592024-02-03T17:27:18.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758000169Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107602024-02-03T17:27:18.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758037184Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107612024-02-03T17:27:18.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.75807586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107622024-02-03T17:27:18.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758118865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107632024-02-03T17:27:18.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758159869Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107642024-02-03T17:27:18.594Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758199339Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107652024-02-03T17:27:18.594Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758242976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107662024-02-03T17:27:18.594Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758288495Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107672024-02-03T17:27:18.594Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.75836635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107682024-02-03T17:27:18.594Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758411815Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107692024-02-03T17:27:18.595Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.758482634Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107702024-02-03T17:27:18.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.759949367Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107712024-02-03T17:27:18.595Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.760039446Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107722024-02-03T17:27:18.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.760073892Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107732024-02-03T17:27:18.595Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.760115806Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107742024-02-03T17:27:18.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.760156748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107752024-02-03T17:27:18.595Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.760197436Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107762024-02-03T17:27:18.599Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.761139277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107772024-02-03T17:27:18.599Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.761457355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107782024-02-03T17:27:18.599Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.761557496Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107792024-02-03T17:27:18.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.76202096Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107802024-02-03T17:27:18.599Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762088949Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107812024-02-03T17:27:18.599Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762116471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107822024-02-03T17:27:18.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762510752Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107832024-02-03T17:27:18.600Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762570598Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107842024-02-03T17:27:18.600Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762595954Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107852024-02-03T17:27:18.600Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762616525Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107862024-02-03T17:27:18.600Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762657627Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107872024-02-03T17:27:18.600Z{"msg":"[0]R flush_numbers[0..12]: [173, 167, 163, 174, 176, 177, 177, 178, 168, 177, 156, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762683018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107882024-02-03T17:27:18.600Z{"msg":"[0]R generation[0..12]: [97, 94, 91, 98, 98, 98, 98, 99, 94, 98, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762706047Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107892024-02-03T17:27:18.600Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.76274822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107902024-02-03T17:27:18.600Z{"msg":"[1]R flush_numbers[0..12]: [173, 167, 163, 174, 176, 177, 177, 178, 168, 177, 156, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762773426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107912024-02-03T17:27:18.600Z{"msg":"[1]R generation[0..12]: [97, 94, 91, 98, 98, 98, 98, 99, 94, 98, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762796085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107922024-02-03T17:27:18.600Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762838477Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107932024-02-03T17:27:18.600Z{"msg":"[2]R flush_numbers[0..12]: [173, 167, 163, 174, 176, 177, 177, 178, 168, 177, 156, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.76286408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107942024-02-03T17:27:18.600Z{"msg":"[2]R generation[0..12]: [97, 94, 91, 98, 98, 98, 98, 99, 94, 98, 87, 95]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762886448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107952024-02-03T17:27:18.600Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762911077Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107962024-02-03T17:27:18.600Z{"msg":"Wait for a query_work_queue command to finish before sending IO
107972024-02-03T17:27:18.600ZMax found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762936115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107982024-02-03T17:27:18.600Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.762997194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
107992024-02-03T17:27:18.600Z{"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763020156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108002024-02-03T17:27:18.600Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763041654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108012024-02-03T17:27:18.600Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763063472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108022024-02-03T17:27:18.600Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763083209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108032024-02-03T17:27:18.600Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.76310437Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108042024-02-03T17:27:18.600Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763125904Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108052024-02-03T17:27:18.600Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763147095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108062024-02-03T17:27:18.601Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763168119Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108072024-02-03T17:27:18.601Z{"msg":"4ace36dd-526c-44e9-be7f-5ea21bf80fe1 is now active with session: 6f15c59e-c01f-4db9-bb5a-d927b131fdd1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763229979Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108082024-02-03T17:27:18.601Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763254857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535,"session_id":"6f15c59e-c01f-4db9-bb5a-d927b131fdd1"}
108092024-02-03T17:27:18.601Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:17.763274927Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535}
108102024-02-03T17:27:18.601ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108112024-02-03T17:27:18.601ZLoading write count information from file "/tmp/test_repair_verify.data"
108122024-02-03T17:27:18.601ZRun Repair workload
108132024-02-03T17:27:18.601Z01/30 Write block 56 len 24576 data: 9 7 8 9 8 7
108142024-02-03T17:27:18.604Z02/30 Read block 551 len 32768
108152024-02-03T17:27:18.604Z03/30 Write block 168 len 32768 data: 5 9 9 8 7 7 8 9
108162024-02-03T17:27:18.604Z04/30 Flush
108172024-02-03T17:27:18.607Z05/30 Write block 220 len 20480 data: 8 13 13 11 12
108182024-02-03T17:27:18.610Z06/30 Write block 116 len 36864 data: 12 13 11 11 10 10 7 8 5
108192024-02-03T17:27:18.610Z07/30 Read block 256 len 4096
108202024-02-03T17:27:18.614Z08/30 Write block 459 len 16384 data: 10 8 7 9
108212024-02-03T17:27:18.614Z09/30 Write block 576 len 12288 data: 7 8 10
108222024-02-03T17:27:18.614Z10/30 Write block 135 len 32768 data: 13 14 15 16 13 9 9 7
108232024-02-03T17:27:18.617Z11/30 Read block 311 len 24576
108242024-02-03T17:27:18.617Z12/30 Read block 133 len 16384
108252024-02-03T17:27:18.621Z13/30 Read block 150 len 32768
108262024-02-03T17:27:18.621Z14/30 Flush
108272024-02-03T17:27:18.624Z15/30 Write block 499 len 36864 data: 10 11 11 11 12 12 16 16 16
108282024-02-03T17:27:18.627Z16/30 Read block 240 len 20480
108292024-02-03T17:27:18.627Z17/30 Write block 296 len 8192 data: 8 9
108302024-02-03T17:27:18.633Z18/30 Write block 528 len 20480 data: 11 11 9 7 7
108312024-02-03T17:27:18.633Z19/30 Write block 437 len 28672 data: 7 7 8 10 11 6 7
108322024-02-03T17:27:18.633Z20/30 Read block 211 len 40960
108332024-02-03T17:27:18.636Z21/30 Write block 235 len 12288 data: 11 10 10
108342024-02-03T17:27:18.636Z22/30 Read block 546 len 40960
108352024-02-03T17:27:18.637Z23/30 Read block 429 len 32768
108362024-02-03T17:27:18.640Z24/30 Flush
108372024-02-03T17:27:18.643Z25/30 Write block 566 len 8192 data: 10 10
108382024-02-03T17:27:18.643Z26/30 Read block 169 len 36864
108392024-02-03T17:27:18.646Z27/30 Read block 20 len 20480
108402024-02-03T17:27:18.646Z28/30 Write block 509 len 36864 data: 16 12 9 7 5 4 6 4 5
108412024-02-03T17:27:18.646Z29/30 Read block 295 len 16384
108422024-02-03T17:27:18.649Z30/30 Write block 97 len 28672 data: 7 5 5 7 7 7 8
108432024-02-03T17:27:18.655Z----------------------------------------------------------------
108442024-02-03T17:27:18.655Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:26
108452024-02-03T17:27:18.655ZStates: Active Active Active
108462024-02-03T17:27:18.656ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108472024-02-03T17:27:18.656Z 5 Acked 1004 Write 5 Done Done Done false
108482024-02-03T17:27:18.656Z 6 Acked 1005 Write 9 Done Done Sent false
108492024-02-03T17:27:18.656Z 7 Acked 1006 Read 1 Done Done Sent false
108502024-02-03T17:27:18.656Z 8 Acked 1007 Write 4 Done Done Sent false
108512024-02-03T17:27:18.657Z 9 Acked 1008 Write 3 Done Done Sent false
108522024-02-03T17:27:18.657Z 10 Acked 1009 Write 8 Done Done Sent false
108532024-02-03T17:27:18.657Z 11 Acked 1010 Read 6 Done Done Sent false
108542024-02-03T17:27:18.657Z 12 Acked 1011 Read 4 Done Done Sent false
108552024-02-03T17:27:18.657Z 13 Acked 1012 Read 8 Done Done Sent false
108562024-02-03T17:27:18.657Z 14 Acked 1013 Flush 0 Done Done Sent false
108572024-02-03T17:27:18.657Z 15 Acked 1014 Write 9 Done Done Sent false
108582024-02-03T17:27:18.657Z 16 Acked 1015 Read 5 Done Done Sent false
108592024-02-03T17:27:18.657Z 17 Acked 1016 Write 2 Done Done Sent false
108602024-02-03T17:27:18.658Z 18 Acked 1017 Write 5 Done Done Sent false
108612024-02-03T17:27:18.658Z 19 Acked 1018 Write 7 Done Done Sent false
108622024-02-03T17:27:18.658Z 20 Acked 1019 Read 10 Done Done Sent false
108632024-02-03T17:27:18.658Z 21 Acked 1020 Write 3 Done Done Sent false
108642024-02-03T17:27:18.658Z 22 Acked 1021 Read 10 Done Done Sent false
108652024-02-03T17:27:18.658Z 23 Acked 1022 Read 8 Done Done Sent false
108662024-02-03T17:27:18.658Z 24 Acked 1023 Flush 0 Done Done Sent false
108672024-02-03T17:27:18.658Z 25 Acked 1024 Write 2 Done Done Sent false
108682024-02-03T17:27:18.659Z 26 Acked 1025 Read 9 Done Done Sent false
108692024-02-03T17:27:18.659Z 27 Acked 1026 Read 5 Done Done Sent false
108702024-02-03T17:27:18.659Z 28 Acked 1027 Write 9 Done Done Sent false
108712024-02-03T17:27:18.659Z 29 Acked 1028 Read 4 Done Done Sent false
108722024-02-03T17:27:18.659Z 30 Acked 1029 Write 7 Sent Sent Sent false
108732024-02-03T17:27:18.659Z STATES DS:0 DS:1 DS:2 TOTAL
108742024-02-03T17:27:18.659Z New 0 0 0 0
108752024-02-03T17:27:18.659Z Sent 1 1 25 27
108762024-02-03T17:27:18.659Z Done 25 25 1 51
108772024-02-03T17:27:18.659Z Skipped 0 0 0 0
108782024-02-03T17:27:18.660Z Error 0 0 0 0
108792024-02-03T17:27:18.660ZLast Flush: 1023 1023 1003
108802024-02-03T17:27:18.660ZDownstairs last five completed: 1003 1002 1001 1000
108812024-02-03T17:27:18.660ZUpstairs last five completed: 30 29 28 27 26
108822024-02-03T17:27:18.660Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:17.817512224Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1535}
108832024-02-03T17:27:18.660ZWrote out file "/tmp/test_repair_verify.data"
108842024-02-03T17:27:19.075Z
108852024-02-03T17:27:19.076ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108862024-02-03T17:27:19.076ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108872024-02-03T17:27:19.076Z 0 000-019 97 97 97 173 173 173 F F F
108882024-02-03T17:27:19.076Z 1 020-039 94 94 94 167 167 167 F F F
108892024-02-03T17:27:19.076Z 2 040-059 100 100 100 179 179 179 F F F
108902024-02-03T17:27:19.077Z 3 060-079 100 100 100 179 179 179 F F F
108912024-02-03T17:27:19.077Z 4 080-099 98 98 98 176 176 176 T T F <---
108922024-02-03T17:27:19.077Z 5 100-119 100 100 98 180 180 177 T T F <---
108932024-02-03T17:27:19.077Z 6 120-139 100 100 98 180 180 177 F F F <---
108942024-02-03T17:27:19.077Z 7 140-159 100 100 99 180 180 178 F F F <---
108952024-02-03T17:27:19.078Z 8 160-179 100 100 100 179 179 179 F F F
108962024-02-03T17:27:19.078Z 9 180-199 98 98 98 177 177 177 F F F
108972024-02-03T17:27:19.078Z 10 200-219 87 87 87 156 156 156 F F F
108982024-02-03T17:27:19.078Z 11 220-239 100 100 95 181 181 169 F F T <---
108992024-02-03T17:27:19.078Z 12 240-259 94 94 94 168 168 168 F F F
109002024-02-03T17:27:19.078Z 13 260-279 97 97 97 173 173 173 F F F
109012024-02-03T17:27:19.078Z 14 280-299 100 100 98 181 181 176 F F F <---
109022024-02-03T17:27:19.078Z 15 300-319 94 94 94 167 167 167 F F F
109032024-02-03T17:27:19.079Z 16 320-339 97 97 97 173 173 173 F F F
109042024-02-03T17:27:19.079Z 17 340-359 97 97 97 173 173 173 F F F
109052024-02-03T17:27:19.079Z 18 360-379 97 97 97 173 173 173 F F F
109062024-02-03T17:27:19.079Z 19 380-399 97 97 97 173 173 173 F F F
109072024-02-03T17:27:19.079Z 20 400-419 92 92 92 164 164 164 F F F
109082024-02-03T17:27:19.079Z 21 420-439 100 100 99 181 181 178 F F F <---
109092024-02-03T17:27:19.080Z 22 440-459 100 100 98 181 181 176 F F F <---
109102024-02-03T17:27:19.080Z 23 460-479 100 100 98 180 180 174 F F F <---
109112024-02-03T17:27:19.080Z 24 480-499 100 100 90 181 181 161 F F F <---
109122024-02-03T17:27:19.080Z 25 500-519 100 100 98 181 181 176 T T F <---
109132024-02-03T17:27:19.080Z 26 520-539 100 100 99 181 181 178 F F F <---
109142024-02-03T17:27:19.080Z 27 540-559 97 97 97 173 173 173 F F F
109152024-02-03T17:27:19.081Z 28 560-579 100 100 97 180 180 173 T T F <---
109162024-02-03T17:27:19.081Z 29 580-599 95 95 95 169 169 169 F F F
109172024-02-03T17:27:19.081ZMax gen: 100, Max flush: 181
109182024-02-03T17:27:19.081ZError: Difference in extent metadata found!
109192024-02-03T17:27:19.081ZA Difference in extent metadata is expected here
109202024-02-03T17:27:19.081ZOn loop 50
109212024-02-03T17:27:19.082Z
109222024-02-03T17:27:19.082ZVerifying data now
109232024-02-03T17:27:23.841ZNo change after verify
109242024-02-03T17:27:23.841ZLoop: 50 Downstairs dump after verify (and repair):
109252024-02-03T17:27:23.868ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109262024-02-03T17:27:23.869Z 0 000-019 97 97 97 173 173 173 F F F
109272024-02-03T17:27:23.869Z 1 020-039 94 94 94 167 167 167 F F F
109282024-02-03T17:27:23.869Z 2 040-059 100 100 100 179 179 179 F F F
109292024-02-03T17:27:23.869Z 3 060-079 100 100 100 179 179 179 F F F
109302024-02-03T17:27:23.869Z 4 080-099 101 101 101 182 182 182 F F F
109312024-02-03T17:27:23.869Z 5 100-119 101 101 101 182 182 182 F F F
109322024-02-03T17:27:23.869Z 6 120-139 100 100 100 180 180 180 F F F
109332024-02-03T17:27:23.872Z 7 140-159 100 100 100 180 180 180 F F F
109342024-02-03T17:27:23.872Z 8 160-179 100 100 100 179 179 179 F F F
109352024-02-03T17:27:23.872Z 9 180-199 98 98 98 177 177 177 F F F
109362024-02-03T17:27:23.872Z 10 200-219 87 87 87 156 156 156 F F F
109372024-02-03T17:27:23.872Z 11 220-239 100 100 100 181 181 181 F F F
109382024-02-03T17:27:23.872Z 12 240-259 94 94 94 168 168 168 F F F
109392024-02-03T17:27:23.872Z 13 260-279 97 97 97 173 173 173 F F F
109402024-02-03T17:27:23.872Z 14 280-299 100 100 100 181 181 181 F F F
109412024-02-03T17:27:23.873Z 15 300-319 94 94 94 167 167 167 F F F
109422024-02-03T17:27:23.873Z 16 320-339 97 97 97 173 173 173 F F F
109432024-02-03T17:27:23.873Z 17 340-359 97 97 97 173 173 173 F F F
109442024-02-03T17:27:23.873Z 18 360-379 97 97 97 173 173 173 F F F
109452024-02-03T17:27:23.873Z 19 380-399 97 97 97 173 173 173 F F F
109462024-02-03T17:27:23.873Z 20 400-419 92 92 92 164 164 164 F F F
109472024-02-03T17:27:23.873Z 21 420-439 100 100 100 181 181 181 F F F
109482024-02-03T17:27:23.873Z 22 440-459 100 100 100 181 181 181 F F F
109492024-02-03T17:27:23.873Z 23 460-479 100 100 100 180 180 180 F F F
109502024-02-03T17:27:23.873Z 24 480-499 100 100 100 181 181 181 F F F
109512024-02-03T17:27:23.873Z 25 500-519 101 101 101 182 182 182 F F F
109522024-02-03T17:27:23.873Z 26 520-539 100 100 100 181 181 181 F F F
109532024-02-03T17:27:23.873Z 27 540-559 97 97 97 173 173 173 F F F
109542024-02-03T17:27:23.873Z 28 560-579 101 101 101 182 182 182 F F F
109552024-02-03T17:27:23.873Z 29 580-599 95 95 95 169 169 169 F F F
109562024-02-03T17:27:23.873ZMax gen: 101, Max flush: 182
109572024-02-03T17:27:23.873Z
109582024-02-03T17:27:23.873ZBegin loop 51
109592024-02-03T17:27:23.873ZDownstairs to restart: 2
109602024-02-03T17:27:23.903Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.070991188Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543}
109612024-02-03T17:27:23.908Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.071586928Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543}
109622024-02-03T17:27:23.908Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.071665282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543}
109632024-02-03T17:27:23.908Z{"msg":"Crucible e827ff9f-585b-4c0e-9afd-bd0de5f7d312 has session id: bcb27ec8-d01f-44aa-9483-92981a1922fd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.071705609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109642024-02-03T17:27:23.908Z{"msg":"Upstairs opts: Upstairs UUID: e827ff9f-585b-4c0e-9afd-bd0de5f7d312, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.07178737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109652024-02-03T17:27:23.908Z{"msg":"Crucible stats registered with UUID: e827ff9f-585b-4c0e-9afd-bd0de5f7d312"Crucible runtime is spawned
109662024-02-03T17:27:23.908Z,"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.071957873Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109672024-02-03T17:27:23.908Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072048052Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543}
109682024-02-03T17:27:23.908Z{"msg":"e827ff9f-585b-4c0e-9afd-bd0de5f7d312 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.07219045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109692024-02-03T17:27:23.908Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072242722Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109702024-02-03T17:27:23.908Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072286594Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109712024-02-03T17:27:23.908Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072329111Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109722024-02-03T17:27:23.908Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072428851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109732024-02-03T17:27:23.908Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.07247399Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109742024-02-03T17:27:23.908Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072554492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109752024-02-03T17:27:23.908Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072623271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109762024-02-03T17:27:23.908Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072801826Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109772024-02-03T17:27:23.908Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072900943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109782024-02-03T17:27:23.908Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072944734Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109792024-02-03T17:27:23.908Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072970588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109802024-02-03T17:27:23.908Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.072993681Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109812024-02-03T17:27:23.908Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.074016187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109822024-02-03T17:27:23.908Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.074073169Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109832024-02-03T17:27:23.909Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.074099545Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109842024-02-03T17:27:23.909Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.074179544Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109852024-02-03T17:27:23.914Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.075177479Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109862024-02-03T17:27:23.914Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.075261976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109872024-02-03T17:27:23.914Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.07587561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109882024-02-03T17:27:23.914Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.07595521Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109892024-02-03T17:27:23.914Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.076275622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109902024-02-03T17:27:23.914Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.076347065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109912024-02-03T17:27:23.915Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.076374729Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109922024-02-03T17:27:23.915Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.076423629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109932024-02-03T17:27:23.915Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.076448259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109942024-02-03T17:27:23.915Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.076506658Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109952024-02-03T17:27:23.915Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.077000715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109962024-02-03T17:27:23.915Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.07772846Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109972024-02-03T17:27:23.915Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.077802484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109982024-02-03T17:27:23.915Z{"msg":"[0]R flush_numbers[0..12]: [173, 167, 179, 179, 182, 182, 180, 180, 179, 177, 156, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.077833834Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
109992024-02-03T17:27:23.915Z{"msg":"[0]R generation[0..12]: [97, 94, 100, 100, 101, 101, 100, 100, 100, 98, 87, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.077878268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110002024-02-03T17:27:23.915Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.077904336Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110012024-02-03T17:27:23.915Z{"msg":"[1]R flush_numbers[0..12]: [173, 167, 179, 179, 182, 182, 180, 180, 179, 177, 156, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.077946151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110022024-02-03T17:27:23.915Z{"msg":"[1]R generation[0..12]: [97, 94, 100, 100, 101, 101, 100, 100, 100, 98, 87, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.077971166Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110032024-02-03T17:27:23.915Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078016607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110042024-02-03T17:27:23.915Z{"msg":"[2]R flush_numbers[0..12]: [173, 167, 179, 179, 182, 182, 180, 180, 179, 177, 156, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078041857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110052024-02-03T17:27:23.915Z{"msg":"[2]R generation[0..12]: [97, 94, 100, 100, 101, 101, 100, 100, 100, 98, 87, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078064516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110062024-02-03T17:27:23.915Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078107688Z","hostname":"Wait for a query_work_queue command to finish before sending IO
110072024-02-03T17:27:23.915Zip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110082024-02-03T17:27:23.915Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078135322Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110092024-02-03T17:27:23.915Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078157547Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110102024-02-03T17:27:23.915Z{"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078178597Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110112024-02-03T17:27:23.915Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078201161Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110122024-02-03T17:27:23.915Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.07822304Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110132024-02-03T17:27:23.915Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078262556Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110142024-02-03T17:27:23.915Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078285862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110152024-02-03T17:27:23.916Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078326542Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110162024-02-03T17:27:23.916Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078350095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110172024-02-03T17:27:23.916Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078370264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110182024-02-03T17:27:23.916Z{"msg":"e827ff9f-585b-4c0e-9afd-bd0de5f7d312 is now active with session: bcb27ec8-d01f-44aa-9483-92981a1922fd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078413695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110192024-02-03T17:27:23.916Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.07843505Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543,"session_id":"bcb27ec8-d01f-44aa-9483-92981a1922fd"}
110202024-02-03T17:27:23.916Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:23.078455158Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543}
110212024-02-03T17:27:23.916ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110222024-02-03T17:27:23.916ZLoading write count information from file "/tmp/test_repair_verify.data"
110232024-02-03T17:27:23.916ZRun Repair workload
110242024-02-03T17:27:23.922Z01/30 Write block 158 len 40960 data: 8 10 10 11 11 9 7 6 5 6
110252024-02-03T17:27:23.922Z02/30 Read block 510 len 4096
110262024-02-03T17:27:23.922Z03/30 Read block 365 len 24576
110272024-02-03T17:27:23.922Z04/30 Flush
110282024-02-03T17:27:23.925Z05/30 Write block 155 len 36864 data: 10 9 9 9 11 11 12 12 10
110292024-02-03T17:27:23.925Z06/30 Read block 187 len 36864
110302024-02-03T17:27:23.925Z07/30 Write block 549 len 4096 data: 10
110312024-02-03T17:27:23.928Z08/30 Read block 372 len 40960
110322024-02-03T17:27:23.928Z09/30 Read block 162 len 36864
110332024-02-03T17:27:23.931Z10/30 Read block 208 len 8192
110342024-02-03T17:27:23.931Z11/30 Read block 90 len 16384
110352024-02-03T17:27:23.934Z12/30 Write block 377 len 32768 data: 8 9 8 8 8 6 6 6
110362024-02-03T17:27:23.934Z13/30 Read block 285 len 36864
110372024-02-03T17:27:23.936Z14/30 Read block 582 len 8192
110382024-02-03T17:27:23.936Z15/30 Read block 0 len 28672
110392024-02-03T17:27:23.939Z16/30 Write block 501 len 28672 data: 12 12 13 13 17 17 17
110402024-02-03T17:27:23.939Z17/30 Read block 576 len 36864
110412024-02-03T17:27:23.942Z18/30 Read block 364 len 12288
110422024-02-03T17:27:23.942Z19/30 Read block 571 len 20480
110432024-02-03T17:27:23.945Z20/30 Write block 277 len 28672 data: 13 14 11 9 11 12 11
110442024-02-03T17:27:23.948Z21/30 Write block 24 len 28672 data: 9 10 8 8 9 7 5
110452024-02-03T17:27:23.948Z22/30 Write block 151 len 12288 data: 10 6 7
110462024-02-03T17:27:23.948Z23/30 Write block 42 len 8192 data: 7 8
110472024-02-03T17:27:23.948Z24/30 Read block 494 len 20480
110482024-02-03T17:27:23.951Z25/30 Write block 463 len 4096 data: 10
110492024-02-03T17:27:23.951Z26/30 Write block 455 len 4096 data: 4
110502024-02-03T17:27:23.951Z27/30 Read block 553 len 16384
110512024-02-03T17:27:23.954Z28/30 Write block 63 len 36864 data: 8 8 6 7 6 3 4 3 3
110522024-02-03T17:27:23.954Z29/30 Read block 226 len 12288
110532024-02-03T17:27:23.958Z30/30 Read block 107 len 20480
110542024-02-03T17:27:23.958Z----------------------------------------------------------------
110552024-02-03T17:27:23.958Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110562024-02-03T17:27:23.958ZStates: Active Active Active
110572024-02-03T17:27:23.958ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110582024-02-03T17:27:23.958Z 1 Acked 1000 Write 10 Done Done Done false
110592024-02-03T17:27:23.958Z 2 Acked 1001 Read 1 Done Done Done false
110602024-02-03T17:27:23.958Z 3 Acked 1002 Read 6 Done Done Sent false
110612024-02-03T17:27:23.958Z 4 Acked 1003 Flush 0 Done Done Sent false
110622024-02-03T17:27:23.958Z 5 Acked 1004 Write 9 Done Done Sent false
110632024-02-03T17:27:23.958Z 6 Acked 1005 Read 9 Done Done Sent false
110642024-02-03T17:27:23.958Z 7 Acked 1006 Write 1 Done Done Sent false
110652024-02-03T17:27:23.958Z 8 Acked 1007 Read 10 Done Done Sent false
110662024-02-03T17:27:23.958Z 9 Acked 1008 Read 9 Done Done Sent false
110672024-02-03T17:27:23.958Z 10 Acked 1009 Read 2 Done Done Sent false
110682024-02-03T17:27:23.958Z 11 Acked 1010 Read 4 Done Done Sent false
110692024-02-03T17:27:23.959Z 12 Acked 1011 Write 8 Done Done Sent false
110702024-02-03T17:27:23.959Z 13 Acked 1012 Read 9 Done Done Sent false
110712024-02-03T17:27:23.959Z 14 Acked 1013 Read 2 Done Done Sent false
110722024-02-03T17:27:23.959Z 15 Acked 1014 Read 7 Done Done Sent false
110732024-02-03T17:27:23.959Z 16 Acked 1015 Write 7 Done Done Sent false
110742024-02-03T17:27:23.959Z 17 Acked 1016 Read 9 Done Done Sent false
110752024-02-03T17:27:23.959Z 18 Acked 1017 Read 3 Done Done Sent false
110762024-02-03T17:27:23.959Z 19 Acked 1018 Read 5 Done Done Sent false
110772024-02-03T17:27:23.959Z 20 Acked 1019 Write 7 Done Done Sent false
110782024-02-03T17:27:23.959Z 21 Acked 1020 Write 7 Done Done Sent false
110792024-02-03T17:27:23.959Z 22 Acked 1021 Write 3 Done Done Sent false
110802024-02-03T17:27:23.959Z 23 Acked 1022 Write 2 Done Done Sent false
110812024-02-03T17:27:23.959Z 24 Acked 1023 Read 5 Done Done Sent false
110822024-02-03T17:27:23.959Z 25 Acked 1024 Write 1 Done Done Sent false
110832024-02-03T17:27:23.959Z 26 Acked 1025 Write 1 Done Done Sent false
110842024-02-03T17:27:23.959Z 27 Acked 1026 Read 4 Done Done Sent false
110852024-02-03T17:27:23.959Z 28 Acked 1027 Write 9 Done Done Sent false
110862024-02-03T17:27:23.959Z 29 Acked 1028 Read 3 Done Done Sent false
110872024-02-03T17:27:23.959Z 30 Acked 1029 Read 5 Done Sent Sent false
110882024-02-03T17:27:23.959Z STATES DS:0 DS:1 DS:2 TOTAL
110892024-02-03T17:27:23.959Z New 0 0 0 0
110902024-02-03T17:27:23.959Z Sent 0 1 28 29
110912024-02-03T17:27:23.959Z Done 30 29 2 61
110922024-02-03T17:27:23.959Z Skipped 0 0 0 0
110932024-02-03T17:27:23.959Z Error 0 0 0 0
110942024-02-03T17:27:23.960ZLast Flush: 1003 1003 0
110952024-02-03T17:27:23.960ZDownstairs last five completed:
110962024-02-03T17:27:23.960ZUpstairs last five completed: 30 29 28 27 26
110972024-02-03T17:27:23.960Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:23.125077696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1543}
110982024-02-03T17:27:23.963ZWrote out file "/tmp/test_repair_verify.data"
110992024-02-03T17:27:23.966Z
111002024-02-03T17:27:23.972ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111012024-02-03T17:27:24.005ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111022024-02-03T17:27:24.005Z 0 000-019 97 97 97 173 173 173 F F F
111032024-02-03T17:27:24.005Z 1 020-039 94 94 94 167 167 167 T T F <---
111042024-02-03T17:27:24.005Z 2 040-059 100 100 100 179 179 179 T T F <---
111052024-02-03T17:27:24.005Z 3 060-079 100 100 100 179 179 179 T T F <---
111062024-02-03T17:27:24.005Z 4 080-099 101 101 101 182 182 182 F F F
111072024-02-03T17:27:24.005Z 5 100-119 101 101 101 182 182 182 F F F
111082024-02-03T17:27:24.005Z 6 120-139 100 100 100 180 180 180 F F F
111092024-02-03T17:27:24.009Z 7 140-159 102 102 100 183 183 180 T T T <---
111102024-02-03T17:27:24.009Z 8 160-179 102 102 100 183 183 179 T T T <---
111112024-02-03T17:27:24.009Z 9 180-199 98 98 98 177 177 177 F F F
111122024-02-03T17:27:24.009Z 10 200-219 87 87 87 156 156 156 F F F
111132024-02-03T17:27:24.009Z 11 220-239 100 100 100 181 181 181 F F F
111142024-02-03T17:27:24.009Z 12 240-259 94 94 94 168 168 168 F F F
111152024-02-03T17:27:24.009Z 13 260-279 97 97 97 173 173 173 T T F <---
111162024-02-03T17:27:24.009Z 14 280-299 100 100 100 181 181 181 T T F <---
111172024-02-03T17:27:24.009Z 15 300-319 94 94 94 167 167 167 F F F
111182024-02-03T17:27:24.009Z 16 320-339 97 97 97 173 173 173 F F F
111192024-02-03T17:27:24.009Z 17 340-359 97 97 97 173 173 173 F F F
111202024-02-03T17:27:24.009Z 18 360-379 97 97 97 173 173 173 T T F <---
111212024-02-03T17:27:24.009Z 19 380-399 97 97 97 173 173 173 T T F <---
111222024-02-03T17:27:24.009Z 20 400-419 92 92 92 164 164 164 F F F
111232024-02-03T17:27:24.009Z 21 420-439 100 100 100 181 181 181 F F F
111242024-02-03T17:27:24.009Z 22 440-459 100 100 100 181 181 181 T T F <---
111252024-02-03T17:27:24.009Z 23 460-479 100 100 100 180 180 180 T T F <---
111262024-02-03T17:27:24.010Z 24 480-499 100 100 100 181 181 181 F F F
111272024-02-03T17:27:24.010Z 25 500-519 101 101 101 182 182 182 T T F <---
111282024-02-03T17:27:24.010Z 26 520-539 100 100 100 181 181 181 F F F
111292024-02-03T17:27:24.010Z 27 540-559 97 97 97 173 173 173 T T F <---
111302024-02-03T17:27:24.010Z 28 560-579 101 101 101 182 182 182 F F F
111312024-02-03T17:27:24.010Z 29 580-599 95 95 95 169 169 169 F F F
111322024-02-03T17:27:24.010ZMax gen: 102, Max flush: 183
111332024-02-03T17:27:24.010ZError: Difference in extent metadata found!
111342024-02-03T17:27:24.013ZA Difference in extent metadata is expected here
111352024-02-03T17:27:24.013ZOn loop 51
111362024-02-03T17:27:24.013Z
111372024-02-03T17:27:24.019ZVerifying data now
111382024-02-03T17:27:30.181ZNo change after verify
111392024-02-03T17:27:30.181ZLoop: 51 Downstairs dump after verify (and repair):
111402024-02-03T17:27:30.207ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111412024-02-03T17:27:30.207Z 0 000-019 97 97 97 173 173 173 F F F
111422024-02-03T17:27:30.207Z 1 020-039 103 103 103 184 184 184 F F F
111432024-02-03T17:27:30.207Z 2 040-059 103 103 103 184 184 184 F F F
111442024-02-03T17:27:30.208Z 3 060-079 103 103 103 184 184 184 F F F
111452024-02-03T17:27:30.211Z 4 080-099 101 101 101 182 182 182 F F F
111462024-02-03T17:27:30.211Z 5 100-119 101 101 101 182 182 182 F F F
111472024-02-03T17:27:30.211Z 6 120-139 100 100 100 180 180 180 F F F
111482024-02-03T17:27:30.211Z 7 140-159 103 103 103 184 184 184 F F F
111492024-02-03T17:27:30.211Z 8 160-179 103 103 103 184 184 184 F F F
111502024-02-03T17:27:30.211Z 9 180-199 98 98 98 177 177 177 F F F
111512024-02-03T17:27:30.211Z 10 200-219 87 87 87 156 156 156 F F F
111522024-02-03T17:27:30.211Z 11 220-239 100 100 100 181 181 181 F F F
111532024-02-03T17:27:30.211Z 12 240-259 94 94 94 168 168 168 F F F
111542024-02-03T17:27:30.211Z 13 260-279 103 103 103 184 184 184 F F F
111552024-02-03T17:27:30.211Z 14 280-299 103 103 103 184 184 184 F F F
111562024-02-03T17:27:30.211Z 15 300-319 94 94 94 167 167 167 F F F
111572024-02-03T17:27:30.211Z 16 320-339 97 97 97 173 173 173 F F F
111582024-02-03T17:27:30.211Z 17 340-359 97 97 97 173 173 173 F F F
111592024-02-03T17:27:30.212Z 18 360-379 103 103 103 184 184 184 F F F
111602024-02-03T17:27:30.212Z 19 380-399 103 103 103 184 184 184 F F F
111612024-02-03T17:27:30.212Z 20 400-419 92 92 92 164 164 164 F F F
111622024-02-03T17:27:30.212Z 21 420-439 100 100 100 181 181 181 F F F
111632024-02-03T17:27:30.212Z 22 440-459 103 103 103 184 184 184 F F F
111642024-02-03T17:27:30.212Z 23 460-479 103 103 103 184 184 184 F F F
111652024-02-03T17:27:30.212Z 24 480-499 100 100 100 181 181 181 F F F
111662024-02-03T17:27:30.212Z 25 500-519 103 103 103 184 184 184 F F F
111672024-02-03T17:27:30.212Z 26 520-539 100 100 100 181 181 181 F F F
111682024-02-03T17:27:30.212Z 27 540-559 103 103 103 184 184 184 F F F
111692024-02-03T17:27:30.212Z 28 560-579 101 101 101 182 182 182 F F F
111702024-02-03T17:27:30.212Z 29 580-599 95 95 95 169 169 169 F F F
111712024-02-03T17:27:30.212ZMax gen: 103, Max flush: 184
111722024-02-03T17:27:30.215Z
111732024-02-03T17:27:30.215ZBegin loop 52
111742024-02-03T17:27:30.215ZDownstairs to restart: 1
111752024-02-03T17:27:30.248Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.415185532Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551}
111762024-02-03T17:27:30.248Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.415732684Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551}
111772024-02-03T17:27:30.248Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.415761623Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551}
111782024-02-03T17:27:30.252Z{"msg":"Crucible 08b6c955-3fc6-47e6-8af4-635597429027 has session id: 7704aee1-a5db-4e3f-86a4-1f94bf203268","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.415892551Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111792024-02-03T17:27:30.252ZCrucible runtime is spawned
111802024-02-03T17:27:30.252Z{"msg":"Upstairs opts: Upstairs UUID: 08b6c955-3fc6-47e6-8af4-635597429027, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416077259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111812024-02-03T17:27:30.252Z{"msg":"Crucible stats registered with UUID: 08b6c955-3fc6-47e6-8af4-635597429027","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416140979Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111822024-02-03T17:27:30.252Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416186449Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551}
111832024-02-03T17:27:30.252Z{"msg":"08b6c955-3fc6-47e6-8af4-635597429027 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416320983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111842024-02-03T17:27:30.252Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416508798Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111852024-02-03T17:27:30.252Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.41661105Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111862024-02-03T17:27:30.253Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416672737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111872024-02-03T17:27:30.253Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416716089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111882024-02-03T17:27:30.253Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416754443Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111892024-02-03T17:27:30.253Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416842157Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111902024-02-03T17:27:30.253Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416888777Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111912024-02-03T17:27:30.253Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416933606Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111922024-02-03T17:27:30.253Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.416976662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111932024-02-03T17:27:30.253Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:29.417023929Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111942024-02-03T17:27:30.253Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:29.417074383Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111952024-02-03T17:27:30.253Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.41712241Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111962024-02-03T17:27:30.253Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.417165888Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111972024-02-03T17:27:30.253Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:29.417242529Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111982024-02-03T17:27:30.253Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.417282412Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
111992024-02-03T17:27:30.253Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.417384134Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112002024-02-03T17:27:30.253Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.417449838Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112012024-02-03T17:27:30.253Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.418351258Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112022024-02-03T17:27:30.253Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.418428011Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112032024-02-03T17:27:30.253Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.418471621Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112042024-02-03T17:27:30.253Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.418533489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112052024-02-03T17:27:30.257Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.419508355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112062024-02-03T17:27:30.257Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.419601234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112072024-02-03T17:27:30.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.42070553Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112082024-02-03T17:27:30.257Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.420796282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112092024-02-03T17:27:30.257Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.4208669Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112102024-02-03T17:27:30.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.420908067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112112024-02-03T17:27:30.257Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.420949445Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112122024-02-03T17:27:30.257Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:29.420990588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112132024-02-03T17:27:31.250Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.417799353Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112142024-02-03T17:27:31.250Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.417928019Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112152024-02-03T17:27:31.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.419181628Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112162024-02-03T17:27:31.256Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.419247075Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112172024-02-03T17:27:31.256Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.419974406Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112182024-02-03T17:27:31.256Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.42088345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112192024-02-03T17:27:31.256Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.420941375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112202024-02-03T17:27:31.256Z{"msg":"[0]R flush_numbers[0..12]: [173, 184, 184, 184, 182, 182, 180, 184, 184, 177, 156, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.420968137Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112212024-02-03T17:27:31.256Z{"msg":"[0]R generation[0..12]: [97, 103, 103, 103, 101, 101, 100, 103, 103, 98, 87, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.420991983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112222024-02-03T17:27:31.256Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.42103536Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112232024-02-03T17:27:31.256Z{"msg":"[1]R flush_numbers[0..12]: [173, 184, 184, 184, 182, 182, 180, 184, 184, 177, 156, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421062169Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112242024-02-03T17:27:31.256Z{"msg":"[1]R generation[0..12]: [97, 103, 103, 103, 101, 101, 100, 103, 103, 98, 87, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421085514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112252024-02-03T17:27:31.256Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421128055Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112262024-02-03T17:27:31.256Z{"msg":"[2]R flush_numbers[0..12]: [173, 184, 184, 184, 182, 182, 180, 184, 184, 177, 156, 181]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421153848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112272024-02-03T17:27:31.256Z{"msg":"[2]R generation[0..12]: [97, 103, 103, 103, 101, 101, 100, 103, 103, 98, 87, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421195586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112282024-02-03T17:27:31.256Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.42122102Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112292024-02-03T17:27:31.256Z{"msg":"Max found gen is 104","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
112302024-02-03T17:27:31.256Zcrucible","level":30,"time":"2024-02-03T17:27:30.421265928Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112312024-02-03T17:27:31.256Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421291114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112322024-02-03T17:27:31.256Z{"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421357571Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112332024-02-03T17:27:31.256Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421383304Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112342024-02-03T17:27:31.256Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421431892Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112352024-02-03T17:27:31.256Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421453962Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112362024-02-03T17:27:31.256Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421502441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112372024-02-03T17:27:31.256Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421525518Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112382024-02-03T17:27:31.257Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421547522Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112392024-02-03T17:27:31.257Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421574326Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112402024-02-03T17:27:31.257Z{"msg":"08b6c955-3fc6-47e6-8af4-635597429027 is now active with session: 7704aee1-a5db-4e3f-86a4-1f94bf203268","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.42162139Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112412024-02-03T17:27:31.257Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421643027Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551,"session_id":"7704aee1-a5db-4e3f-86a4-1f94bf203268"}
112422024-02-03T17:27:31.257Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:30.421663384Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551}
112432024-02-03T17:27:31.260ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112442024-02-03T17:27:31.260ZLoading write count information from file "/tmp/test_repair_verify.data"
112452024-02-03T17:27:31.260ZRun Repair workload
112462024-02-03T17:27:31.260Z01/30 Write block 473 len 24576 data: 4 5 5 6 9 10
112472024-02-03T17:27:31.260Z02/30 Write block 438 len 12288 data: 8 9 11
112482024-02-03T17:27:31.260Z03/30 Write block 157 len 4096 data: 10
112492024-02-03T17:27:31.260Z04/30 Write block 257 len 8192 data: 4 4
112502024-02-03T17:27:31.264Z05/30 Read block 53 len 28672
112512024-02-03T17:27:31.264Z06/30 Read block 62 len 32768
112522024-02-03T17:27:31.267Z07/30 Write block 423 len 40960 data: 6 6 6 5 5 4 3 5 5 5
112532024-02-03T17:27:31.267Z08/30 Write block 470 len 4096 data: 7
112542024-02-03T17:27:31.267Z09/30 Flush
112552024-02-03T17:27:31.270Z10/30 Write block 501 len 8192 data: 13 13
112562024-02-03T17:27:31.273Z11/30 Flush
112572024-02-03T17:27:31.279Z12/30 Write block 304 len 16384 data: 8 10 11 8
112582024-02-03T17:27:31.279Z13/30 Read block 573 len 12288
112592024-02-03T17:27:31.279Z14/30 Write block 160 len 36864 data: 12 13 13 11 8 7 6 7 6
112602024-02-03T17:27:31.282Z15/30 Write block 12 len 32768 data: 12 10 7 4 6 5 5 4
112612024-02-03T17:27:31.282Z16/30 Write block 248 len 28672 data: 6 7 12 7 5 4 7
112622024-02-03T17:27:31.282Z17/30 Flush
112632024-02-03T17:27:31.285Z18/30 Flush
112642024-02-03T17:27:31.285Z19/30 Write block 355 len 8192 data: 11 14
112652024-02-03T17:27:31.288Z20/30 Write block 374 len 36864 data: 5 5 8 9 10 9 9 9 7
112662024-02-03T17:27:31.288Z21/30 Write block 227 len 12288 data: 15 13 10
112672024-02-03T17:27:31.288Z22/30 Read block 593 len 24576
112682024-02-03T17:27:31.291Z23/30 Read block 322 len 20480
112692024-02-03T17:27:31.291Z24/30 Flush
112702024-02-03T17:27:31.294Z25/30 Write block 573 len 12288 data: 4 5 5
112712024-02-03T17:27:31.297Z26/30 Write block 56 len 36864 data: 10 8 9 10 9 8 7 9 9
112722024-02-03T17:27:31.297Z27/30 Write block 135 len 32768 data: 14 15 16 17 14 10 10 8
112732024-02-03T17:27:31.297Z28/30 Read block 585 len 4096
112742024-02-03T17:27:31.301Z29/30 Write block 247 len 20480 data: 6 7 8 13 8
112752024-02-03T17:27:31.301Z30/30 Read block 560 len 24576
112762024-02-03T17:27:31.305Z----------------------------------------------------------------
112772024-02-03T17:27:31.305Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
112782024-02-03T17:27:31.305ZStates: Active Active Active
112792024-02-03T17:27:31.306ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112802024-02-03T17:27:31.306Z 1 Acked 1000 Write 6 Done Sent Done false
112812024-02-03T17:27:31.306Z 2 Acked 1001 Write 3 Done Sent Done false
112822024-02-03T17:27:31.306Z 3 Acked 1002 Write 1 Done Sent Done false
112832024-02-03T17:27:31.306Z 4 Acked 1003 Write 2 Done Sent Done false
112842024-02-03T17:27:31.306Z 5 Acked 1004 Read 7 Done Sent Done false
112852024-02-03T17:27:31.306Z 6 Acked 1005 Read 8 Done Sent Done false
112862024-02-03T17:27:31.307Z 7 Acked 1006 Write 10 Done Sent Done false
112872024-02-03T17:27:31.307Z 8 Acked 1007 Write 1 Done Sent Done false
112882024-02-03T17:27:31.307Z 9 Acked 1008 Flush 0 Done Sent Done false
112892024-02-03T17:27:31.307Z 10 Acked 1009 Write 2 Done Sent Done false
112902024-02-03T17:27:31.307Z 11 Acked 1010 Flush 0 Done Sent Done false
112912024-02-03T17:27:31.307Z 12 Acked 1011 Write 4 Done Sent Done false
112922024-02-03T17:27:31.307Z 13 Acked 1012 Read 3 Done Sent Done false
112932024-02-03T17:27:31.307Z 14 Acked 1013 Write 9 Done Sent Done false
112942024-02-03T17:27:31.307Z 15 Acked 1014 Write 8 Done Sent Done false
112952024-02-03T17:27:31.307Z 16 Acked 1015 Write 7 Done Sent Done false
112962024-02-03T17:27:31.307Z 17 Acked 1016 Flush 0 Done Sent Done false
112972024-02-03T17:27:31.307Z 18 Acked 1017 Flush 0 Done Sent Done false
112982024-02-03T17:27:31.307Z 19 Acked 1018 Write 2 Done Sent Done false
112992024-02-03T17:27:31.307Z 20 Acked 1019 Write 9 Done Sent Done false
113002024-02-03T17:27:31.307Z 21 Acked 1020 Write 3 Done Sent Done false
113012024-02-03T17:27:31.308Z 22 Acked 1021 Read 6 Done Sent Done false
113022024-02-03T17:27:31.308Z 23 Acked 1022 Read 5 Done Sent Done false
113032024-02-03T17:27:31.308Z 24 Acked 1023 Flush 0 Done Sent Done false
113042024-02-03T17:27:31.308Z 25 Acked 1024 Write 3 Done Sent Done false
113052024-02-03T17:27:31.308Z 26 Acked 1025 Write 9 Done Sent Done false
113062024-02-03T17:27:31.308Z 27 Acked 1026 Write 8 Done Sent Done false
113072024-02-03T17:27:31.308Z 28 Acked 1027 Read 1 Done Sent Done false
113082024-02-03T17:27:31.308Z 29 Acked 1028 Write 5 Done Sent Done false
113092024-02-03T17:27:31.308Z 30 Acked 1029 Read 6 Done Sent Done false
113102024-02-03T17:27:31.308Z STATES DS:0 DS:1 DS:2 TOTAL
113112024-02-03T17:27:31.308Z New 0 0 0 0
113122024-02-03T17:27:31.308Z Sent 0 30 0 30
113132024-02-03T17:27:31.308Z Done 30 0 30 60
113142024-02-03T17:27:31.308Z Skipped 0 0 0 0
113152024-02-03T17:27:31.309Z Error 0 0 0 0
113162024-02-03T17:27:31.309ZLast Flush: 1023 0 1023
113172024-02-03T17:27:31.309ZDownstairs last five completed:
113182024-02-03T17:27:31.309ZUpstairs last five completed: 30 29 28 27 26
113192024-02-03T17:27:31.309Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:30.470441487Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1551}
113202024-02-03T17:27:31.309ZWrote out file "/tmp/test_repair_verify.data"
113212024-02-03T17:27:31.312Z
113222024-02-03T17:27:31.317ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113232024-02-03T17:27:31.347ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113242024-02-03T17:27:31.347Z 0 000-019 104 97 104 187 173 187 F F F <---
113252024-02-03T17:27:31.354Z 1 020-039 103 103 103 184 184 184 F F F
113262024-02-03T17:27:31.354Z 2 040-059 103 103 103 184 184 184 T F T <---
113272024-02-03T17:27:31.355Z 3 060-079 103 103 103 184 184 184 T F T <---
113282024-02-03T17:27:31.355Z 4 080-099 101 101 101 182 182 182 F F F
113292024-02-03T17:27:31.355Z 5 100-119 101 101 101 182 182 182 F F F
113302024-02-03T17:27:31.355Z 6 120-139 100 100 100 180 180 180 T F T <---
113312024-02-03T17:27:31.355Z 7 140-159 104 103 104 185 184 185 T F T <---
113322024-02-03T17:27:31.355Z 8 160-179 104 103 104 187 184 187 F F F <---
113332024-02-03T17:27:31.355Z 9 180-199 98 98 98 177 177 177 F F F
113342024-02-03T17:27:31.355Z 10 200-219 87 87 87 156 156 156 F F F
113352024-02-03T17:27:31.355Z 11 220-239 104 100 104 189 181 189 F F F <---
113362024-02-03T17:27:31.355Z 12 240-259 104 94 104 187 168 187 T F T <---
113372024-02-03T17:27:31.355Z 13 260-279 103 103 103 184 184 184 F F F
113382024-02-03T17:27:31.355Z 14 280-299 103 103 103 184 184 184 F F F
113392024-02-03T17:27:31.355Z 15 300-319 104 94 104 187 167 187 F F F <---
113402024-02-03T17:27:31.355Z 16 320-339 97 97 97 173 173 173 F F F
113412024-02-03T17:27:31.355Z 17 340-359 104 97 104 189 173 189 F F F <---
113422024-02-03T17:27:31.355Z 18 360-379 104 103 104 189 184 189 F F F <---
113432024-02-03T17:27:31.355Z 19 380-399 104 103 104 189 184 189 F F F <---
113442024-02-03T17:27:31.355Z 20 400-419 92 92 92 164 164 164 F F F
113452024-02-03T17:27:31.355Z 21 420-439 104 100 104 185 181 185 F F F <---
113462024-02-03T17:27:31.355Z 22 440-459 104 103 104 185 184 185 F F F <---
113472024-02-03T17:27:31.355Z 23 460-479 104 103 104 185 184 185 F F F <---
113482024-02-03T17:27:31.355Z 24 480-499 100 100 100 181 181 181 F F F
113492024-02-03T17:27:31.355Z 25 500-519 104 103 104 186 184 186 F F F <---
113502024-02-03T17:27:31.356Z 26 520-539 100 100 100 181 181 181 F F F
113512024-02-03T17:27:31.356Z 27 540-559 103 103 103 184 184 184 F F F
113522024-02-03T17:27:31.356Z 28 560-579 101 101 101 182 182 182 T F T <---
113532024-02-03T17:27:31.356Z 29 580-599 95 95 95 169 169 169 F F F
113542024-02-03T17:27:31.356ZMax gen: 104, Max flush: 189
113552024-02-03T17:27:31.356ZError: Difference in extent metadata found!
113562024-02-03T17:27:31.361ZA Difference in extent metadata is expected here
113572024-02-03T17:27:31.361ZOn loop 52
113582024-02-03T17:27:31.361Z
113592024-02-03T17:27:31.362ZVerifying data now
113602024-02-03T17:27:36.793ZNo change after verify
113612024-02-03T17:27:36.793ZLoop: 52 Downstairs dump after verify (and repair):
113622024-02-03T17:27:36.817ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113632024-02-03T17:27:36.817Z 0 000-019 104 104 104 187 187 187 F F F
113642024-02-03T17:27:36.818Z 1 020-039 103 103 103 184 184 184 F F F
113652024-02-03T17:27:36.818Z 2 040-059 105 105 105 190 190 190 F F F
113662024-02-03T17:27:36.818Z 3 060-079 105 105 105 190 190 190 F F F
113672024-02-03T17:27:36.818Z 4 080-099 101 101 101 182 182 182 F F F
113682024-02-03T17:27:36.821Z 5 100-119 101 101 101 182 182 182 F F F
113692024-02-03T17:27:36.821Z 6 120-139 105 105 105 190 190 190 F F F
113702024-02-03T17:27:36.821Z 7 140-159 105 105 105 190 190 190 F F F
113712024-02-03T17:27:36.821Z 8 160-179 104 104 104 187 187 187 F F F
113722024-02-03T17:27:36.821Z 9 180-199 98 98 98 177 177 177 F F F
113732024-02-03T17:27:36.821Z 10 200-219 87 87 87 156 156 156 F F F
113742024-02-03T17:27:36.821Z 11 220-239 104 104 104 189 189 189 F F F
113752024-02-03T17:27:36.821Z 12 240-259 105 105 105 190 190 190 F F F
113762024-02-03T17:27:36.821Z 13 260-279 103 103 103 184 184 184 F F F
113772024-02-03T17:27:36.822Z 14 280-299 103 103 103 184 184 184 F F F
113782024-02-03T17:27:36.822Z 15 300-319 104 104 104 187 187 187 F F F
113792024-02-03T17:27:36.824Z 16 320-339 97 97 97 173 173 173 F F F
113802024-02-03T17:27:36.824Z 17 340-359 104 104 104 189 189 189 F F F
113812024-02-03T17:27:36.824Z 18 360-379 104 104 104 189 189 189 F F F
113822024-02-03T17:27:36.825Z 19 380-399 104 104 104 189 189 189 F F F
113832024-02-03T17:27:36.825Z 20 400-419 92 92 92 164 164 164 F F F
113842024-02-03T17:27:36.825Z 21 420-439 104 104 104 185 185 185 F F F
113852024-02-03T17:27:36.825Z 22 440-459 104 104 104 185 185 185 F F F
113862024-02-03T17:27:36.825Z 23 460-479 104 104 104 185 185 185 F F F
113872024-02-03T17:27:36.825Z 24 480-499 100 100 100 181 181 181 F F F
113882024-02-03T17:27:36.825Z 25 500-519 104 104 104 186 186 186 F F F
113892024-02-03T17:27:36.825Z 26 520-539 100 100 100 181 181 181 F F F
113902024-02-03T17:27:36.825Z 27 540-559 103 103 103 184 184 184 F F F
113912024-02-03T17:27:36.825Z 28 560-579 105 105 105 190 190 190 F F F
113922024-02-03T17:27:36.825Z 29 580-599 95 95 95 169 169 169 F F F
113932024-02-03T17:27:36.825ZMax gen: 105, Max flush: 190
113942024-02-03T17:27:36.829Z
113952024-02-03T17:27:36.829ZBegin loop 53
113962024-02-03T17:27:36.829ZDownstairs to restart: 2
113972024-02-03T17:27:36.856Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.02403974Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559}
113982024-02-03T17:27:36.857Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.024552034Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559}
113992024-02-03T17:27:36.857Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.024604311Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559}
114002024-02-03T17:27:36.875Z{"msg":"Crucible 070d388e-1c2e-46a4-b460-2e37cb95ebae has session id: 3d47fb56-c0a4-458a-8403-f73818ed1036","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.024645725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559Crucible runtime is spawned
114012024-02-03T17:27:36.875Z,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114022024-02-03T17:27:36.876Z{"msg":"Upstairs opts: Upstairs UUID: 070d388e-1c2e-46a4-b460-2e37cb95ebae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.024850423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114032024-02-03T17:27:36.876Z{"msg":"Crucible stats registered with UUID: 070d388e-1c2e-46a4-b460-2e37cb95ebae","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.024921118Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114042024-02-03T17:27:36.876Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.024948734Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559}
114052024-02-03T17:27:36.876Z{"msg":"070d388e-1c2e-46a4-b460-2e37cb95ebae active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.02500737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114062024-02-03T17:27:36.876Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.025078419Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114072024-02-03T17:27:36.876Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.025127078Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114082024-02-03T17:27:36.876Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.02520498Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114092024-02-03T17:27:36.876Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.025251846Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114102024-02-03T17:27:36.876Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.02534697Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114112024-02-03T17:27:36.876Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.025494438Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114122024-02-03T17:27:36.876Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.025624673Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114132024-02-03T17:27:36.876Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.025834399Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114142024-02-03T17:27:36.877Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.025963744Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114152024-02-03T17:27:36.877Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:36.026027207Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114162024-02-03T17:27:36.877Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:36.026107048Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114172024-02-03T17:27:36.877Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:36.02615242Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114182024-02-03T17:27:36.877Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.026187894Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114192024-02-03T17:27:36.877Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.026273979Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114202024-02-03T17:27:36.877Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.026373865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114212024-02-03T17:27:36.877Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.026416259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114222024-02-03T17:27:36.877Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.02645687Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114232024-02-03T17:27:36.877Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.026788728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114242024-02-03T17:27:36.877Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.026854812Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114252024-02-03T17:27:36.877Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.027140062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114262024-02-03T17:27:36.878Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.027202274Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114272024-02-03T17:27:36.878Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.027827722Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114282024-02-03T17:27:36.878Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.028085084Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114292024-02-03T17:27:36.878Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.028684797Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114302024-02-03T17:27:36.878Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.028832485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114312024-02-03T17:27:36.878Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.028874417Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114322024-02-03T17:27:36.878Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.029012822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114332024-02-03T17:27:36.878Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.029067892Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114342024-02-03T17:27:36.878Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:36.029093091Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114352024-02-03T17:27:37.859Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.026742476Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114362024-02-03T17:27:37.859Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.026873426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114372024-02-03T17:27:37.864Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.028152527Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114382024-02-03T17:27:37.864Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.028213451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114392024-02-03T17:27:37.864Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.02902142Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114402024-02-03T17:27:37.864Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.029928711Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114412024-02-03T17:27:37.864Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.029992272Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114422024-02-03T17:27:37.864Z{"msg":"[0]R flush_numbers[0..12]: [187, 184, 190, 190, 182, 182, 190, 190, 187, 177, 156, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030019739Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114432024-02-03T17:27:37.864Z{"msg":"[0]R generation[0..12]: [104, 103, 105, 105, 101, 101, 105, 105, 104, 98, 87, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030049707Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114442024-02-03T17:27:37.864Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030099038Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114452024-02-03T17:27:37.864Z{"msg":"[1]R flush_numbers[0..12]: [187, 184, 190, 190, 182, 182, 190, 190, 187, 177, 156, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030125777Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114462024-02-03T17:27:37.864Z{"msg":"[1]R generation[0..12]: [104, 103, 105, 105, 101, 101, 105, 105, 104, 98, 87, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030168748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114472024-02-03T17:27:37.865Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030194579Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114482024-02-03T17:27:37.865Z{"msg":"[2]R flush_numbers[0..12]: [187, 184, 190, 190, 182, 182, 190, 190, 187, 177, 156, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030237535Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114492024-02-03T17:27:37.865Z{"msg":"[2]R generation[0..12]: [104, 103, 105, 105, 101, 101, 105, 105, 104, 98, 87, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030262393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114502024-02-03T17:27:37.865Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030304918Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559Wait for a query_work_queue command to finish before sending IO
114512024-02-03T17:27:37.865Z,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114522024-02-03T17:27:37.865Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030333485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114532024-02-03T17:27:37.865Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030383031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114542024-02-03T17:27:37.865Z{"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030406005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114552024-02-03T17:27:37.865Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030450899Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114562024-02-03T17:27:37.865Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030474741Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114572024-02-03T17:27:37.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030514169Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114582024-02-03T17:27:37.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030537562Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114592024-02-03T17:27:37.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030578507Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114602024-02-03T17:27:37.865Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030602282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114612024-02-03T17:27:37.865Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030642023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114622024-02-03T17:27:37.865Z{"msg":"070d388e-1c2e-46a4-b460-2e37cb95ebae is now active with session: 3d47fb56-c0a4-458a-8403-f73818ed1036","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030665692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114632024-02-03T17:27:37.865Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030708069Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559,"session_id":"3d47fb56-c0a4-458a-8403-f73818ed1036"}
114642024-02-03T17:27:37.865Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:37.030753223Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559}
114652024-02-03T17:27:37.869ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114662024-02-03T17:27:37.869ZLoading write count information from file "/tmp/test_repair_verify.data"
114672024-02-03T17:27:37.869ZRun Repair workload
114682024-02-03T17:27:37.869Z01/30 Write block 48 len 4096 data: 8
114692024-02-03T17:27:37.869Z02/30 Flush
114702024-02-03T17:27:37.876Z03/30 Write block 80 len 4096 data: 7
114712024-02-03T17:27:37.876Z04/30 Read block 222 len 12288
114722024-02-03T17:27:37.876Z05/30 Read block 434 len 40960
114732024-02-03T17:27:37.876Z06/30 Write block 542 len 36864 data: 8 9 10 11 10 10 11 11 10
114742024-02-03T17:27:37.876Z07/30 Flush
114752024-02-03T17:27:37.883Z08/30 Write block 441 len 8192 data: 12 7
114762024-02-03T17:27:37.883Z09/30 Read block 429 len 8192
114772024-02-03T17:27:37.888Z10/30 Write block 405 len 20480 data: 11 10 10 7 8
114782024-02-03T17:27:37.888Z11/30 Flush
114792024-02-03T17:27:37.888Z12/30 Write block 157 len 8192 data: 11 10
114802024-02-03T17:27:37.888Z13/30 Write block 218 len 40960 data: 7 8 9 14 14 12 13 13 12 16
114812024-02-03T17:27:37.888Z14/30 Read block 542 len 40960
114822024-02-03T17:27:37.893Z15/30 Read block 365 len 8192
114832024-02-03T17:27:37.893Z16/30 Flush
114842024-02-03T17:27:37.893Z17/30 Write block 420 len 20480 data: 11 8 7 7 7
114852024-02-03T17:27:37.893Z18/30 Read block 248 len 8192
114862024-02-03T17:27:37.899Z19/30 Flush
114872024-02-03T17:27:37.899Z20/30 Write block 507 len 4096 data: 18
114882024-02-03T17:27:37.902Z21/30 Write block 323 len 20480 data: 7 8 8 9 8
114892024-02-03T17:27:37.902Z22/30 Flush
114902024-02-03T17:27:37.902Z23/30 Write block 575 len 28672 data: 6 8 9 11 11 11 10
114912024-02-03T17:27:37.905Z24/30 Flush
114922024-02-03T17:27:37.908Z25/30 Write block 266 len 24576 data: 12 10 12 15 18 17
114932024-02-03T17:27:37.908Z26/30 Write block 418 len 32768 data: 10 11 12 9 8 8 8 7
114942024-02-03T17:27:37.908Z27/30 Read block 552 len 28672
114952024-02-03T17:27:37.915Z28/30 Write block 106 len 12288 data: 8 9 10
114962024-02-03T17:27:37.915Z29/30 Write block 396 len 4096 data: 9
114972024-02-03T17:27:37.915Z30/30 Read block 536 len 36864
114982024-02-03T17:27:37.915Z----------------------------------------------------------------
114992024-02-03T17:27:37.915Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:23
115002024-02-03T17:27:37.915ZStates: Active Active Active
115012024-02-03T17:27:37.915ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
115022024-02-03T17:27:37.915Z 8 Acked 1007 Write 2 Done Done Sent false
115032024-02-03T17:27:37.916Z 9 Acked 1008 Read 2 Done Done Sent false
115042024-02-03T17:27:37.916Z 10 Acked 1009 Write 5 Done Done Sent false
115052024-02-03T17:27:37.916Z 11 Acked 1010 Flush 0 Done Done Sent false
115062024-02-03T17:27:37.916Z 12 Acked 1011 Write 2 Done Done Sent false
115072024-02-03T17:27:37.916Z 13 Acked 1012 Write 10 Done Done Sent false
115082024-02-03T17:27:37.916Z 14 Acked 1013 Read 10 Done Done Sent false
115092024-02-03T17:27:37.916Z 15 Acked 1014 Read 2 Done Done Sent false
115102024-02-03T17:27:37.916Z 16 Acked 1015 Flush 0 Done Done Sent false
115112024-02-03T17:27:37.916Z 17 Acked 1016 Write 5 Done Done Sent false
115122024-02-03T17:27:37.916Z 18 Acked 1017 Read 2 Done Done Sent false
115132024-02-03T17:27:37.916Z 19 Acked 1018 Flush 0 Done Done Sent false
115142024-02-03T17:27:37.916Z 20 Acked 1019 Write 1 Done Done Sent false
115152024-02-03T17:27:37.917Z 21 Acked 1020 Write 5 Done Done Sent false
115162024-02-03T17:27:37.917Z 22 Acked 1021 Flush 0 Done Done Sent false
115172024-02-03T17:27:37.917Z 23 Acked 1022 Write 7 Done Done Sent false
115182024-02-03T17:27:37.917Z 24 Acked 1023 Flush 0 Done Done Sent false
115192024-02-03T17:27:37.917Z 25 Acked 1024 Write 6 Done Done Sent false
115202024-02-03T17:27:37.917Z 26 Acked 1025 Write 8 Done Done Sent false
115212024-02-03T17:27:37.917Z 27 Acked 1026 Read 7 Done Done Sent false
115222024-02-03T17:27:37.917Z 28 Acked 1027 Write 3 Done Done Sent false
115232024-02-03T17:27:37.917Z 29 Acked 1028 Write 1 Done Done Sent false
115242024-02-03T17:27:37.917Z 30 Acked 1029 Read 9 Done Sent Sent false
115252024-02-03T17:27:37.917Z STATES DS:0 DS:1 DS:2 TOTAL
115262024-02-03T17:27:37.917Z New 0 0 0 0
115272024-02-03T17:27:37.918Z Sent 0 1 23 24
115282024-02-03T17:27:37.918Z Done 23 22 0 45
115292024-02-03T17:27:37.918Z Skipped 0 0 0 0
115302024-02-03T17:27:37.918Z Error 0 0 0 0
115312024-02-03T17:27:37.918ZLast Flush: 1023 1023 1006
115322024-02-03T17:27:37.918ZDownstairs last five completed: 1006 1005 1004 1003 1002
115332024-02-03T17:27:37.918ZUpstairs last five completed: 30 29 28 27 26
115342024-02-03T17:27:37.918Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:37.082413592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1559}
115352024-02-03T17:27:37.918ZWrote out file "/tmp/test_repair_verify.data"
115362024-02-03T17:27:37.934Z
115372024-02-03T17:27:37.937ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115382024-02-03T17:27:37.957ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115392024-02-03T17:27:37.957Z 0 000-019 104 104 104 187 187 187 F F F
115402024-02-03T17:27:37.957Z 1 020-039 103 103 103 184 184 184 F F F
115412024-02-03T17:27:37.957Z 2 040-059 106 106 106 191 191 191 F F F
115422024-02-03T17:27:37.957Z 3 060-079 105 105 105 190 190 190 F F F
115432024-02-03T17:27:37.957Z 4 080-099 106 106 106 192 192 192 F F F
115442024-02-03T17:27:37.957Z 5 100-119 101 101 101 182 182 182 T T F <---
115452024-02-03T17:27:37.958Z 6 120-139 105 105 105 190 190 190 F F F
115462024-02-03T17:27:37.958Z 7 140-159 106 106 105 194 194 190 F F F <---
115472024-02-03T17:27:37.964Z 8 160-179 104 104 104 187 187 187 F F F
115482024-02-03T17:27:37.965Z 9 180-199 98 98 98 177 177 177 F F F
115492024-02-03T17:27:37.965Z 10 200-219 106 106 87 194 194 156 F F F <---
115502024-02-03T17:27:37.965Z 11 220-239 106 106 104 194 194 189 F F F <---
115512024-02-03T17:27:37.965Z 12 240-259 105 105 105 190 190 190 F F F
115522024-02-03T17:27:37.965Z 13 260-279 103 103 103 184 184 184 T T F <---
115532024-02-03T17:27:37.965Z 14 280-299 103 103 103 184 184 184 F F F
115542024-02-03T17:27:37.965Z 15 300-319 104 104 104 187 187 187 F F F
115552024-02-03T17:27:37.965Z 16 320-339 106 106 97 196 196 173 F F F <---
115562024-02-03T17:27:37.965Z 17 340-359 104 104 104 189 189 189 F F F
115572024-02-03T17:27:37.965Z 18 360-379 104 104 104 189 189 189 F F F
115582024-02-03T17:27:37.965Z 19 380-399 104 104 104 189 189 189 T T F <---
115592024-02-03T17:27:37.965Z 20 400-419 106 106 92 193 193 164 T T F <---
115602024-02-03T17:27:37.965Z 21 420-439 106 106 104 195 195 185 T T F <---
115612024-02-03T17:27:37.965Z 22 440-459 106 106 104 193 193 185 F F F <---
115622024-02-03T17:27:37.965Z 23 460-479 104 104 104 185 185 185 F F F
115632024-02-03T17:27:37.965Z 24 480-499 100 100 100 181 181 181 F F F
115642024-02-03T17:27:37.965Z 25 500-519 106 106 104 196 196 186 F F F <---
115652024-02-03T17:27:37.965Z 26 520-539 100 100 100 181 181 181 F F F
115662024-02-03T17:27:37.965Z 27 540-559 106 106 106 192 192 192 F F F
115672024-02-03T17:27:37.965Z 28 560-579 106 106 105 197 197 190 F F F <---
115682024-02-03T17:27:37.965Z 29 580-599 106 106 95 197 197 169 F F F <---
115692024-02-03T17:27:37.965ZMax gen: 106, Max flush: 197
115702024-02-03T17:27:37.965ZError: Difference in extent metadata found!
115712024-02-03T17:27:37.966ZA Difference in extent metadata is expected here
115722024-02-03T17:27:37.966ZOn loop 53
115732024-02-03T17:27:37.966Z
115742024-02-03T17:27:37.969ZVerifying data now
115752024-02-03T17:27:43.094ZNo change after verify
115762024-02-03T17:27:43.094ZLoop: 53 Downstairs dump after verify (and repair):
115772024-02-03T17:27:43.118ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115782024-02-03T17:27:43.118Z 0 000-019 104 104 104 187 187 187 F F F
115792024-02-03T17:27:43.121Z 1 020-039 103 103 103 184 184 184 F F F
115802024-02-03T17:27:43.121Z 2 040-059 106 106 106 191 191 191 F F F
115812024-02-03T17:27:43.121Z 3 060-079 105 105 105 190 190 190 F F F
115822024-02-03T17:27:43.121Z 4 080-099 106 106 106 192 192 192 F F F
115832024-02-03T17:27:43.121Z 5 100-119 107 107 107 198 198 198 F F F
115842024-02-03T17:27:43.121Z 6 120-139 105 105 105 190 190 190 F F F
115852024-02-03T17:27:43.122Z 7 140-159 106 106 106 194 194 194 F F F
115862024-02-03T17:27:43.122Z 8 160-179 104 104 104 187 187 187 F F F
115872024-02-03T17:27:43.122Z 9 180-199 98 98 98 177 177 177 F F F
115882024-02-03T17:27:43.122Z 10 200-219 106 106 106 194 194 194 F F F
115892024-02-03T17:27:43.122Z 11 220-239 106 106 106 194 194 194 F F F
115902024-02-03T17:27:43.122Z 12 240-259 105 105 105 190 190 190 F F F
115912024-02-03T17:27:43.122Z 13 260-279 107 107 107 198 198 198 F F F
115922024-02-03T17:27:43.122Z 14 280-299 103 103 103 184 184 184 F F F
115932024-02-03T17:27:43.122Z 15 300-319 104 104 104 187 187 187 F F F
115942024-02-03T17:27:43.122Z 16 320-339 106 106 106 196 196 196 F F F
115952024-02-03T17:27:43.122Z 17 340-359 104 104 104 189 189 189 F F F
115962024-02-03T17:27:43.122Z 18 360-379 104 104 104 189 189 189 F F F
115972024-02-03T17:27:43.122Z 19 380-399 107 107 107 198 198 198 F F F
115982024-02-03T17:27:43.122Z 20 400-419 107 107 107 198 198 198 F F F
115992024-02-03T17:27:43.122Z 21 420-439 107 107 107 198 198 198 F F F
116002024-02-03T17:27:43.122Z 22 440-459 106 106 106 193 193 193 F F F
116012024-02-03T17:27:43.122Z 23 460-479 104 104 104 185 185 185 F F F
116022024-02-03T17:27:43.122Z 24 480-499 100 100 100 181 181 181 F F F
116032024-02-03T17:27:43.122Z 25 500-519 106 106 106 196 196 196 F F F
116042024-02-03T17:27:43.122Z 26 520-539 100 100 100 181 181 181 F F F
116052024-02-03T17:27:43.122Z 27 540-559 106 106 106 192 192 192 F F F
116062024-02-03T17:27:43.122Z 28 560-579 106 106 106 197 197 197 F F F
116072024-02-03T17:27:43.122Z 29 580-599 106 106 106 197 197 197 F F F
116082024-02-03T17:27:43.122ZMax gen: 107, Max flush: 198
116092024-02-03T17:27:43.125Z
116102024-02-03T17:27:43.125ZBegin loop 54
116112024-02-03T17:27:43.126ZDownstairs to restart: 1
116122024-02-03T17:27:43.156Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.324330722Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567}
116132024-02-03T17:27:43.156Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.324808211Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567}
116142024-02-03T17:27:43.156Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.324837406Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567}
116152024-02-03T17:27:43.160Z{"msg":"Crucible 9da5dc3b-a4cc-4d78-af2f-2a41cdb34883 has session id: cc55232a-1569-45d6-b29b-84c6c01a9664","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.324873913Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116162024-02-03T17:27:43.160Z{"msg":"Upstairs opts: Upstairs UUID: 9da5dc3b-a4cc-4d78-af2f-2a41cdb34883, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.324957406Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116172024-02-03T17:27:43.160Z{"msg":"Crucible stats registered with UUID: 9da5dc3b-a4cc-4d78-af2f-2a41cdb34883","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325000483Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116182024-02-03T17:27:43.161ZCrucible runtime is spawned
116192024-02-03T17:27:43.161Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325229696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567}
116202024-02-03T17:27:43.161Z{"msg":"9da5dc3b-a4cc-4d78-af2f-2a41cdb34883 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.32535645Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116212024-02-03T17:27:43.161Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325434113Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116222024-02-03T17:27:43.161Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325483586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116232024-02-03T17:27:43.161Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325555604Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116242024-02-03T17:27:43.161Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.32560189Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116252024-02-03T17:27:43.161Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325697575Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116262024-02-03T17:27:43.161Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325746245Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116272024-02-03T17:27:43.161Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325787031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116282024-02-03T17:27:43.161Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325832052Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116292024-02-03T17:27:43.161Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325918419Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116302024-02-03T17:27:43.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.325996267Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116312024-02-03T17:27:43.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.326043278Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116322024-02-03T17:27:43.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.326089168Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116332024-02-03T17:27:43.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.327577156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116342024-02-03T17:27:43.161Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.327655711Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116352024-02-03T17:27:43.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.327701219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116362024-02-03T17:27:43.161Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.327744777Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116372024-02-03T17:27:43.167Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.328396871Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116382024-02-03T17:27:43.167Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.328475704Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116392024-02-03T17:27:43.167Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.328860354Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116402024-02-03T17:27:43.167Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.328976725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116412024-02-03T17:27:43.167Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.329538603Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116422024-02-03T17:27:43.167Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.329869237Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116432024-02-03T17:27:43.167Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.329932147Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116442024-02-03T17:27:43.167Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.329959073Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116452024-02-03T17:27:43.167Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.32998148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116462024-02-03T17:27:43.167Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330003976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116472024-02-03T17:27:43.167Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330065794Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116482024-02-03T17:27:43.167Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330377914Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116492024-02-03T17:27:43.167Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330447991Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116502024-02-03T17:27:43.167Z{"msg":"[0]R flush_numbers[0..12]: [187, 184, 191, 190, 192, 198, 190, 194, 187, 177, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330477273Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116512024-02-03T17:27:43.167Z{"msg":"[0]R generation[0..12]: [104, 103, 106, 105, 106, 107, 105, 106, 104, 98, 106, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330500745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116522024-02-03T17:27:43.167Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330526897Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116532024-02-03T17:27:43.167Z{"msg":"[1]R flush_numbers[0..12]: [187, 184, 191, 190, 192, 198, 190, 194, 187, 177, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330550503Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116542024-02-03T17:27:43.167Z{"msg":"[1]R generation[0..12]: [104, 103, 106, 105, 106, 107, 105, 106, 104, 98, 106, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330592843Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116552024-02-03T17:27:43.167Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330618514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116562024-02-03T17:27:43.167Z{"msg":"[2]R flush_numbers[0..12]: [187, 184, 191, 190, 192, 198, 190, 194, 187, 177, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330660771Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116572024-02-03T17:27:43.167Z{"msg":"[2]R generation[0..12]: [104, 103, 106, 105, 106, 107, 105, 106, 104, 98, 106, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330685754Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116582024-02-03T17:27:43.167Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
116592024-02-03T17:27:43.167Z,"time":"2024-02-03T17:27:42.330728156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116602024-02-03T17:27:43.168Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.33075566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116612024-02-03T17:27:43.168Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330800307Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116622024-02-03T17:27:43.168Z{"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330823473Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116632024-02-03T17:27:43.168Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330864324Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116642024-02-03T17:27:43.168Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330888327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116652024-02-03T17:27:43.168Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330927136Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116662024-02-03T17:27:43.168Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.330950695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116672024-02-03T17:27:43.168Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.33099803Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116682024-02-03T17:27:43.168Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.331021793Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116692024-02-03T17:27:43.168Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.331088168Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116702024-02-03T17:27:43.168Z{"msg":"9da5dc3b-a4cc-4d78-af2f-2a41cdb34883 is now active with session: cc55232a-1569-45d6-b29b-84c6c01a9664","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.331159497Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116712024-02-03T17:27:43.168Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.331181692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567,"session_id":"cc55232a-1569-45d6-b29b-84c6c01a9664"}
116722024-02-03T17:27:43.168Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:42.331202Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567}
116732024-02-03T17:27:43.168ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116742024-02-03T17:27:43.168ZLoading write count information from file "/tmp/test_repair_verify.data"
116752024-02-03T17:27:43.168ZRun Repair workload
116762024-02-03T17:27:43.168Z01/30 Write block 308 len 12288 data: 9 9 7
116772024-02-03T17:27:43.172Z02/30 Read block 314 len 40960
116782024-02-03T17:27:43.172Z03/30 Read block 353 len 12288
116792024-02-03T17:27:43.172Z04/30 Write block 570 len 8192 data: 7 6
116802024-02-03T17:27:43.172Z05/30 Read block 101 len 20480
116812024-02-03T17:27:43.172Z06/30 Read block 377 len 40960
116822024-02-03T17:27:43.175Z07/30 Read block 126 len 32768
116832024-02-03T17:27:43.177Z08/30 Read block 351 len 12288
116842024-02-03T17:27:43.178Z09/30 Read block 466 len 36864
116852024-02-03T17:27:43.180Z10/30 Write block 405 len 36864 data: 12 11 11 8 9 8 9 8 8
116862024-02-03T17:27:43.181Z11/30 Write block 56 len 40960 data: 11 9 10 11 10 9 8 10 10 7
116872024-02-03T17:27:43.183Z12/30 Flush
116882024-02-03T17:27:43.186Z13/30 Write block 564 len 8192 data: 7 8
116892024-02-03T17:27:43.186Z14/30 Read block 79 len 20480
116902024-02-03T17:27:43.189Z15/30 Read block 216 len 12288
116912024-02-03T17:27:43.189Z16/30 Read block 86 len 40960
116922024-02-03T17:27:43.192Z17/30 Flush
116932024-02-03T17:27:43.195Z18/30 Write block 307 len 28672 data: 9 10 10 8 7 4 6
116942024-02-03T17:27:43.195Z19/30 Read block 376 len 8192
116952024-02-03T17:27:43.197Z20/30 Write block 150 len 28672 data: 10 11 7 8 9 11 10
116962024-02-03T17:27:43.197Z21/30 Write block 420 len 16384 data: 13 10 9 9
116972024-02-03T17:27:43.197Z22/30 Read block 376 len 28672
116982024-02-03T17:27:43.200Z23/30 Write block 288 len 16384 data: 7 7 6 7
116992024-02-03T17:27:43.200Z24/30 Read block 383 len 32768
117002024-02-03T17:27:43.203Z25/30 Read block 536 len 4096
117012024-02-03T17:27:43.203Z26/30 Read block 212 len 4096
117022024-02-03T17:27:43.206Z27/30 Read block 239 len 8192
117032024-02-03T17:27:43.206Z28/30 Write block 15 len 4096 data: 5
117042024-02-03T17:27:43.206Z29/30 Write block 232 len 40960 data: 11 11 11 12 11 11 8 11 7 7
117052024-02-03T17:27:43.209Z30/30 Write block 333 len 24576 data: 8 9 7 7 6 5
117062024-02-03T17:27:43.210Z----------------------------------------------------------------
117072024-02-03T17:27:43.210Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
117082024-02-03T17:27:43.210ZStates: Active Active Active
117092024-02-03T17:27:43.210ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117102024-02-03T17:27:43.210Z 1 Acked 1000 Write 3 Done Done Done false
117112024-02-03T17:27:43.210Z 2 Acked 1001 Read 10 Done Done Done false
117122024-02-03T17:27:43.210Z 3 Acked 1002 Read 3 Done Sent Done false
117132024-02-03T17:27:43.210Z 4 Acked 1003 Write 2 Done Sent Done false
117142024-02-03T17:27:43.210Z 5 Acked 1004 Read 5 Done Sent Done false
117152024-02-03T17:27:43.210Z 6 Acked 1005 Read 10 Done Sent Done false
117162024-02-03T17:27:43.210Z 7 Acked 1006 Read 8 Done Sent Done false
117172024-02-03T17:27:43.210Z 8 Acked 1007 Read 3 Done Sent Done false
117182024-02-03T17:27:43.210Z 9 Acked 1008 Read 9 Done Sent Done false
117192024-02-03T17:27:43.210Z 10 Acked 1009 Write 9 Done Sent Done false
117202024-02-03T17:27:43.210Z 11 Acked 1010 Write 10 Done Sent Done false
117212024-02-03T17:27:43.210Z 12 Acked 1011 Flush 0 Done Sent Done false
117222024-02-03T17:27:43.210Z 13 Acked 1012 Write 2 Done Sent Done false
117232024-02-03T17:27:43.210Z 14 Acked 1013 Read 5 Done Sent Done false
117242024-02-03T17:27:43.210Z 15 Acked 1014 Read 3 Done Sent Done false
117252024-02-03T17:27:43.210Z 16 Acked 1015 Read 10 Done Sent Done false
117262024-02-03T17:27:43.210Z 17 Acked 1016 Flush 0 Done Sent Done false
117272024-02-03T17:27:43.210Z 18 Acked 1017 Write 7 Done Sent Done false
117282024-02-03T17:27:43.210Z 19 Acked 1018 Read 2 Done Sent Done false
117292024-02-03T17:27:43.210Z 20 Acked 1019 Write 7 Done Sent Done false
117302024-02-03T17:27:43.211Z 21 Acked 1020 Write 4 Done Sent Done false
117312024-02-03T17:27:43.211Z 22 Acked 1021 Read 7 Done Sent Done false
117322024-02-03T17:27:43.211Z 23 Acked 1022 Write 4 Done Sent Done false
117332024-02-03T17:27:43.211Z 24 Acked 1023 Read 8 Done Sent Done false
117342024-02-03T17:27:43.211Z 25 Acked 1024 Read 1 Done Sent Done false
117352024-02-03T17:27:43.211Z 26 Acked 1025 Read 1 Done Sent Done false
117362024-02-03T17:27:43.211Z 27 Acked 1026 Read 2 Done Sent Done false
117372024-02-03T17:27:43.211Z 28 Acked 1027 Write 1 Done Sent Done false
117382024-02-03T17:27:43.211Z 29 Acked 1028 Write 10 Done Sent Done false
117392024-02-03T17:27:43.211Z 30 Acked 1029 Write 6 Sent Sent Sent false
117402024-02-03T17:27:43.211Z STATES DS:0 DS:1 DS:2 TOTAL
117412024-02-03T17:27:43.211Z New 0 0 0 0
117422024-02-03T17:27:43.211Z Sent 1 28 1 30
117432024-02-03T17:27:43.211Z Done 29 2 29 60
117442024-02-03T17:27:43.211Z Skipped 0 0 0 0
117452024-02-03T17:27:43.211Z Error 0 0 0 0
117462024-02-03T17:27:43.211ZLast Flush: 1016 0 1016
117472024-02-03T17:27:43.211ZDownstairs last five completed:
117482024-02-03T17:27:43.211ZUpstairs last five completed: 30 29 28 27 26
117492024-02-03T17:27:43.211Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:42.377477401Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1567}
117502024-02-03T17:27:43.211ZWrote out file "/tmp/test_repair_verify.data"
117512024-02-03T17:27:43.217Z
117522024-02-03T17:27:43.225ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117532024-02-03T17:27:43.256ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117542024-02-03T17:27:43.256Z 0 000-019 104 104 104 187 187 187 T F T <---
117552024-02-03T17:27:43.256Z 1 020-039 103 103 103 184 184 184 F F F
117562024-02-03T17:27:43.256Z 2 040-059 108 106 108 199 191 199 F F F <---
117572024-02-03T17:27:43.259Z 3 060-079 108 105 108 199 190 199 F F F <---
117582024-02-03T17:27:43.259Z 4 080-099 106 106 106 192 192 192 F F F
117592024-02-03T17:27:43.259Z 5 100-119 107 107 107 198 198 198 F F F
117602024-02-03T17:27:43.259Z 6 120-139 105 105 105 190 190 190 F F F
117612024-02-03T17:27:43.259Z 7 140-159 106 106 106 194 194 194 T F T <---
117622024-02-03T17:27:43.259Z 8 160-179 104 104 104 187 187 187 F F F
117632024-02-03T17:27:43.259Z 9 180-199 98 98 98 177 177 177 F F F
117642024-02-03T17:27:43.259Z 10 200-219 106 106 106 194 194 194 F F F
117652024-02-03T17:27:43.259Z 11 220-239 106 106 106 194 194 194 T F T <---
117662024-02-03T17:27:43.259Z 12 240-259 105 105 105 190 190 190 T F T <---
117672024-02-03T17:27:43.259Z 13 260-279 107 107 107 198 198 198 F F F
117682024-02-03T17:27:43.259Z 14 280-299 103 103 103 184 184 184 T F T <---
117692024-02-03T17:27:43.260Z 15 300-319 108 104 108 199 187 199 T T T <---
117702024-02-03T17:27:43.260Z 16 320-339 106 106 106 196 196 196 T F T <---
117712024-02-03T17:27:43.260Z 17 340-359 104 104 104 189 189 189 F F F
117722024-02-03T17:27:43.260Z 18 360-379 104 104 104 189 189 189 F F F
117732024-02-03T17:27:43.260Z 19 380-399 107 107 107 198 198 198 F F F
117742024-02-03T17:27:43.260Z 20 400-419 108 107 108 199 198 199 F F F <---
117752024-02-03T17:27:43.260Z 21 420-439 107 107 107 198 198 198 T F T <---
117762024-02-03T17:27:43.260Z 22 440-459 106 106 106 193 193 193 F F F
117772024-02-03T17:27:43.260Z 23 460-479 104 104 104 185 185 185 F F F
117782024-02-03T17:27:43.260Z 24 480-499 100 100 100 181 181 181 F F F
117792024-02-03T17:27:43.260Z 25 500-519 106 106 106 196 196 196 F F F
117802024-02-03T17:27:43.260Z 26 520-539 100 100 100 181 181 181 F F F
117812024-02-03T17:27:43.260Z 27 540-559 106 106 106 192 192 192 F F F
117822024-02-03T17:27:43.260Z 28 560-579 108 106 108 200 197 200 F F F <---
117832024-02-03T17:27:43.260Z 29 580-599 106 106 106 197 197 197 F F F
117842024-02-03T17:27:43.260ZMax gen: 108, Max flush: 200
117852024-02-03T17:27:43.260ZError: Difference in extent metadata found!
117862024-02-03T17:27:43.263ZA Difference in extent metadata is expected here
117872024-02-03T17:27:43.264ZOn loop 54
117882024-02-03T17:27:43.264Z
117892024-02-03T17:27:43.269ZVerifying data now
117902024-02-03T17:27:49.360ZNo change after verify
117912024-02-03T17:27:49.360ZLoop: 54 Downstairs dump after verify (and repair):
117922024-02-03T17:27:49.367ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117932024-02-03T17:27:49.368Z 0 000-019 109 109 109 201 201 201 F F F
117942024-02-03T17:27:49.371Z 1 020-039 103 103 103 184 184 184 F F F
117952024-02-03T17:27:49.371Z 2 040-059 108 108 108 199 199 199 F F F
117962024-02-03T17:27:49.371Z 3 060-079 108 108 108 199 199 199 F F F
117972024-02-03T17:27:49.371Z 4 080-099 106 106 106 192 192 192 F F F
117982024-02-03T17:27:49.371Z 5 100-119 107 107 107 198 198 198 F F F
117992024-02-03T17:27:49.371Z 6 120-139 105 105 105 190 190 190 F F F
118002024-02-03T17:27:49.371Z 7 140-159 109 109 109 201 201 201 F F F
118012024-02-03T17:27:49.372Z 8 160-179 104 104 104 187 187 187 F F F
118022024-02-03T17:27:49.372Z 9 180-199 98 98 98 177 177 177 F F F
118032024-02-03T17:27:49.372Z 10 200-219 106 106 106 194 194 194 F F F
118042024-02-03T17:27:49.372Z 11 220-239 109 109 109 201 201 201 F F F
118052024-02-03T17:27:49.372Z 12 240-259 109 109 109 201 201 201 F F F
118062024-02-03T17:27:49.372Z 13 260-279 107 107 107 198 198 198 F F F
118072024-02-03T17:27:49.372Z 14 280-299 109 109 109 201 201 201 F F F
118082024-02-03T17:27:49.372Z 15 300-319 109 109 109 201 201 201 F F F
118092024-02-03T17:27:49.372Z 16 320-339 109 109 109 201 201 201 F F F
118102024-02-03T17:27:49.372Z 17 340-359 104 104 104 189 189 189 F F F
118112024-02-03T17:27:49.372Z 18 360-379 104 104 104 189 189 189 F F F
118122024-02-03T17:27:49.372Z 19 380-399 107 107 107 198 198 198 F F F
118132024-02-03T17:27:49.372Z 20 400-419 108 108 108 199 199 199 F F F
118142024-02-03T17:27:49.372Z 21 420-439 109 109 109 201 201 201 F F F
118152024-02-03T17:27:49.372Z 22 440-459 106 106 106 193 193 193 F F F
118162024-02-03T17:27:49.372Z 23 460-479 104 104 104 185 185 185 F F F
118172024-02-03T17:27:49.372Z 24 480-499 100 100 100 181 181 181 F F F
118182024-02-03T17:27:49.372Z 25 500-519 106 106 106 196 196 196 F F F
118192024-02-03T17:27:49.372Z 26 520-539 100 100 100 181 181 181 F F F
118202024-02-03T17:27:49.372Z 27 540-559 106 106 106 192 192 192 F F F
118212024-02-03T17:27:49.372Z 28 560-579 108 108 108 200 200 200 F F F
118222024-02-03T17:27:49.372Z 29 580-599 106 106 106 197 197 197 F F F
118232024-02-03T17:27:49.372ZMax gen: 109, Max flush: 201
118242024-02-03T17:27:49.376Z
118252024-02-03T17:27:49.376ZBegin loop 55
118262024-02-03T17:27:49.376ZDownstairs to restart: 1
118272024-02-03T17:27:49.405Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.572897409Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575}
118282024-02-03T17:27:49.410Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.573474582Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575}
118292024-02-03T17:27:49.410Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
118302024-02-03T17:27:49.410Z,"time":"2024-02-03T17:27:48.573554306Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575}
118312024-02-03T17:27:49.410Z{"msg":"Crucible 912249d1-77a2-4c36-8f07-e5dc7d62f58f has session id: 9ea29530-0d8a-4063-9847-96e1da5c6359","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.573654021Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118322024-02-03T17:27:49.410Z{"msg":"Upstairs opts: Upstairs UUID: 912249d1-77a2-4c36-8f07-e5dc7d62f58f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.573745628Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118332024-02-03T17:27:49.410Z{"msg":"Crucible stats registered with UUID: 912249d1-77a2-4c36-8f07-e5dc7d62f58f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.573777272Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118342024-02-03T17:27:49.410Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.573820767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575}
118352024-02-03T17:27:49.410Z{"msg":"912249d1-77a2-4c36-8f07-e5dc7d62f58f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.573926223Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118362024-02-03T17:27:49.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.573981529Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118372024-02-03T17:27:49.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574008243Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118382024-02-03T17:27:49.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574032186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118392024-02-03T17:27:49.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574096403Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118402024-02-03T17:27:49.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574123228Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118412024-02-03T17:27:49.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574145601Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118422024-02-03T17:27:49.411Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.5741802Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118432024-02-03T17:27:49.411Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574378736Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118442024-02-03T17:27:49.411Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574438923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118452024-02-03T17:27:49.411Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574620076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118462024-02-03T17:27:49.411Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574702119Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118472024-02-03T17:27:49.411Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.574747518Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118482024-02-03T17:27:49.411Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.576251371Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118492024-02-03T17:27:49.411Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.576336344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118502024-02-03T17:27:49.411Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.576437064Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118512024-02-03T17:27:49.411Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.576497084Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118522024-02-03T17:27:49.411Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.576535398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118532024-02-03T17:27:49.412Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.576577702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118542024-02-03T17:27:49.417Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.577400562Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118552024-02-03T17:27:49.417Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.577494966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118562024-02-03T17:27:49.417Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.577920282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118572024-02-03T17:27:49.417Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578543123Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118582024-02-03T17:27:49.417Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578614785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118592024-02-03T17:27:49.417Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578642291Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118602024-02-03T17:27:49.417Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578667306Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118612024-02-03T17:27:49.418Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578738628Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118622024-02-03T17:27:49.418Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578802889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118632024-02-03T17:27:49.418Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578828144Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118642024-02-03T17:27:49.418Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578870243Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118652024-02-03T17:27:49.418Z{"msg":"[0]R flush_numbers[0..12]: [201, 184, 199, 199, 192, 198, 190, 201, 187, 177, 194, 201]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578895966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118662024-02-03T17:27:49.418Z{"msg":"[0]R generation[0..12]: [109, 103, 108, 108, 106, 107, 105, 109, 104, 98, 106, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578937973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118672024-02-03T17:27:49.418Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.578963825Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118682024-02-03T17:27:49.418Z{"msg":"[1]R flush_numbers[0..12]: [201, 184, 199, 199, 192, 198, 190, 201, 187, 177, 194, 201]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579007089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118692024-02-03T17:27:49.418Z{"msg":"[1]R generation[0..12]: [109, 103, 108, 108, 106, 107, 105, 109, 104, 98, 106, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579031774Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118702024-02-03T17:27:49.418Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579075003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118712024-02-03T17:27:49.418Z{"msg":"[2]R flush_numbers[0..12]: [201, 184, 199, 199, 192, 198, 190, 201, 187, 177, 194, 201]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.57910078Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118722024-02-03T17:27:49.418Z{"msg":"Wait for a query_work_queue command to finish before sending IO
118732024-02-03T17:27:49.418Z[2]R generation[0..12]: [109, 103, 108, 108, 106, 107, 105, 109, 104, 98, 106, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579146129Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118742024-02-03T17:27:49.418Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579171604Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118752024-02-03T17:27:49.419Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579193717Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118762024-02-03T17:27:49.419Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579227977Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118772024-02-03T17:27:49.419Z{"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579264828Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118782024-02-03T17:27:49.419Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579310544Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118792024-02-03T17:27:49.419Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579335488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118802024-02-03T17:27:49.419Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579355353Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118812024-02-03T17:27:49.419Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579397273Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118822024-02-03T17:27:49.419Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.57944471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118832024-02-03T17:27:49.419Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579468397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118842024-02-03T17:27:49.419Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579488757Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118852024-02-03T17:27:49.419Z{"msg":"912249d1-77a2-4c36-8f07-e5dc7d62f58f is now active with session: 9ea29530-0d8a-4063-9847-96e1da5c6359","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579537063Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118862024-02-03T17:27:49.419Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579558706Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575,"session_id":"9ea29530-0d8a-4063-9847-96e1da5c6359"}
118872024-02-03T17:27:49.419Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:48.579578746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575}
118882024-02-03T17:27:49.420ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118892024-02-03T17:27:49.420ZLoading write count information from file "/tmp/test_repair_verify.data"
118902024-02-03T17:27:49.420ZRun Repair workload
118912024-02-03T17:27:49.420Z01/30 Write block 356 len 24576 data: 15 14 14 15 14 14
118922024-02-03T17:27:49.420Z02/30 Read block 89 len 12288
118932024-02-03T17:27:49.424Z03/30 Write block 114 len 36864 data: 14 13 13 14 12 12 11 11 8
118942024-02-03T17:27:49.424Z04/30 Write block 526 len 28672 data: 11 12 12 12 10 8 8
118952024-02-03T17:27:49.424Z05/30 Write block 413 len 12288 data: 9 8 4
118962024-02-03T17:27:49.424Z06/30 Read block 269 len 8192
118972024-02-03T17:27:49.424Z07/30 Write block 269 len 4096 data: 16
118982024-02-03T17:27:49.424Z08/30 Write block 375 len 20480 data: 6 9 10 11 10
118992024-02-03T17:27:49.424Z09/30 Read block 379 len 40960
119002024-02-03T17:27:49.428Z10/30 Read block 282 len 4096
119012024-02-03T17:27:49.428Z11/30 Write block 418 len 28672 data: 11 12 14 11 10 10 9
119022024-02-03T17:27:49.428Z12/30 Write block 438 len 12288 data: 9 10 12
119032024-02-03T17:27:49.431Z13/30 Write block 542 len 16384 data: 9 10 11 12
119042024-02-03T17:27:49.431Z14/30 Read block 427 len 8192
119052024-02-03T17:27:49.431Z15/30 Write block 389 len 8192 data: 5 5
119062024-02-03T17:27:49.432Z16/30 Read block 9 len 20480
119072024-02-03T17:27:49.435Z17/30 Read block 156 len 20480
119082024-02-03T17:27:49.436Z18/30 Write block 387 len 20480 data: 6 6 6 6 6
119092024-02-03T17:27:49.436Z19/30 Read block 299 len 32768
119102024-02-03T17:27:49.439Z20/30 Read block 278 len 16384
119112024-02-03T17:27:49.439Z21/30 Read block 131 len 24576
119122024-02-03T17:27:49.442Z22/30 Write block 70 len 32768 data: 4 4 3 4 4 5 6 8
119132024-02-03T17:27:49.442Z23/30 Read block 475 len 16384
119142024-02-03T17:27:49.445Z24/30 Write block 481 len 20480 data: 10 9 8 6 6
119152024-02-03T17:27:49.445Z25/30 Read block 8 len 12288
119162024-02-03T17:27:49.450Z26/30 Write block 348 len 24576 data: 8 8 8 8 6 8
119172024-02-03T17:27:49.450Z27/30 Write block 383 len 12288 data: 7 7 6
119182024-02-03T17:27:49.450Z28/30 Write block 244 len 8192 data: 3 4
119192024-02-03T17:27:49.450Z29/30 Write block 139 len 32768 data: 15 11 11 9 4 4 5 4
119202024-02-03T17:27:49.450Z30/30 Read block 299 len 28672
119212024-02-03T17:27:49.455Z----------------------------------------------------------------
119222024-02-03T17:27:49.455Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
119232024-02-03T17:27:49.455ZStates: Active Active Active
119242024-02-03T17:27:49.455ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119252024-02-03T17:27:49.455Z 1 Acked 1000 Write 6 Done Done Done false
119262024-02-03T17:27:49.455Z 2 Acked 1001 Read 3 Done Done Done false
119272024-02-03T17:27:49.455Z 3 Acked 1002 Write 9 Done Done Done false
119282024-02-03T17:27:49.455Z 4 Acked 1003 Write 7 Done Done Done false
119292024-02-03T17:27:49.455Z 5 Acked 1004 Write 3 Done Sent Done false
119302024-02-03T17:27:49.455Z 6 Acked 1005 Read 2 Done Sent Done false
119312024-02-03T17:27:49.455Z 7 Acked 1006 Write 1 Done Sent Done false
119322024-02-03T17:27:49.455Z 8 Acked 1007 Write 5 Done Sent Done false
119332024-02-03T17:27:49.456Z 9 Acked 1008 Read 10 Done Sent Done false
119342024-02-03T17:27:49.456Z 10 Acked 1009 Read 1 Done Sent Done false
119352024-02-03T17:27:49.456Z 11 Acked 1010 Write 7 Done Sent Done false
119362024-02-03T17:27:49.456Z 12 Acked 1011 Write 3 Done Sent Done false
119372024-02-03T17:27:49.456Z 13 Acked 1012 Write 4 Done Sent Done false
119382024-02-03T17:27:49.456Z 14 Acked 1013 Read 2 Done Sent Done false
119392024-02-03T17:27:49.456Z 15 Acked 1014 Write 2 Done Sent Done false
119402024-02-03T17:27:49.456Z 16 Acked 1015 Read 5 Done Sent Done false
119412024-02-03T17:27:49.456Z 17 Acked 1016 Read 5 Done Sent Done false
119422024-02-03T17:27:49.456Z 18 Acked 1017 Write 5 Done Sent Done false
119432024-02-03T17:27:49.456Z 19 Acked 1018 Read 8 Done Sent Done false
119442024-02-03T17:27:49.456Z 20 Acked 1019 Read 4 Done Sent Done false
119452024-02-03T17:27:49.456Z 21 Acked 1020 Read 6 Done Sent Done false
119462024-02-03T17:27:49.456Z 22 Acked 1021 Write 8 Done Sent Done false
119472024-02-03T17:27:49.456Z 23 Acked 1022 Read 4 Done Sent Done false
119482024-02-03T17:27:49.456Z 24 Acked 1023 Write 5 Done Sent Done false
119492024-02-03T17:27:49.456Z 25 Acked 1024 Read 3 Done Sent Done false
119502024-02-03T17:27:49.456Z 26 Acked 1025 Write 6 Done Sent Done false
119512024-02-03T17:27:49.456Z 27 Acked 1026 Write 3 Done Sent Done false
119522024-02-03T17:27:49.457Z 28 Acked 1027 Write 2 Done Sent Done false
119532024-02-03T17:27:49.457Z 29 Acked 1028 Write 8 Done Sent Done false
119542024-02-03T17:27:49.457Z 30 Acked 1029 Read 7 Done Sent Sent false
119552024-02-03T17:27:49.457Z STATES DS:0 DS:1 DS:2 TOTAL
119562024-02-03T17:27:49.457Z New 0 0 0 0
119572024-02-03T17:27:49.457Z Sent 0 26 1 27
119582024-02-03T17:27:49.457Z Done 30 4 29 63
119592024-02-03T17:27:49.457Z Skipped 0 0 0 0
119602024-02-03T17:27:49.457Z Error 0 0 0 0
119612024-02-03T17:27:49.457ZLast Flush: 0 0 0
119622024-02-03T17:27:49.457ZDownstairs last five completed:
119632024-02-03T17:27:49.457ZUpstairs last five completed: 30 29 28 27 26
119642024-02-03T17:27:49.457Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:48.621446857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1575}
119652024-02-03T17:27:49.457ZWrote out file "/tmp/test_repair_verify.data"
119662024-02-03T17:27:49.461Z
119672024-02-03T17:27:49.468ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119682024-02-03T17:27:49.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119692024-02-03T17:27:49.502Z 0 000-019 109 109 109 201 201 201 F F F
119702024-02-03T17:27:49.502Z 1 020-039 103 103 103 184 184 184 F F F
119712024-02-03T17:27:49.502Z 2 040-059 108 108 108 199 199 199 F F F
119722024-02-03T17:27:49.502Z 3 060-079 108 108 108 199 199 199 T F T <---
119732024-02-03T17:27:49.505Z 4 080-099 106 106 106 192 192 192 F F F
119742024-02-03T17:27:49.505Z 5 100-119 107 107 107 198 198 198 T T T
119752024-02-03T17:27:49.505Z 6 120-139 105 105 105 190 190 190 T T T
119762024-02-03T17:27:49.505Z 7 140-159 109 109 109 201 201 201 T F T <---
119772024-02-03T17:27:49.505Z 8 160-179 104 104 104 187 187 187 F F F
119782024-02-03T17:27:49.505Z 9 180-199 98 98 98 177 177 177 F F F
119792024-02-03T17:27:49.505Z 10 200-219 106 106 106 194 194 194 F F F
119802024-02-03T17:27:49.506Z 11 220-239 109 109 109 201 201 201 F F F
119812024-02-03T17:27:49.506Z 12 240-259 109 109 109 201 201 201 T F T <---
119822024-02-03T17:27:49.506Z 13 260-279 107 107 107 198 198 198 T F T <---
119832024-02-03T17:27:49.506Z 14 280-299 109 109 109 201 201 201 F F F
119842024-02-03T17:27:49.506Z 15 300-319 109 109 109 201 201 201 F F F
119852024-02-03T17:27:49.506Z 16 320-339 109 109 109 201 201 201 F F F
119862024-02-03T17:27:49.506Z 17 340-359 104 104 104 189 189 189 T T T
119872024-02-03T17:27:49.506Z 18 360-379 104 104 104 189 189 189 T T T
119882024-02-03T17:27:49.506Z 19 380-399 107 107 107 198 198 198 T F T <---
119892024-02-03T17:27:49.506Z 20 400-419 108 108 108 199 199 199 T F T <---
119902024-02-03T17:27:49.506Z 21 420-439 109 109 109 201 201 201 T F T <---
119912024-02-03T17:27:49.506Z 22 440-459 106 106 106 193 193 193 T F T <---
119922024-02-03T17:27:49.506Z 23 460-479 104 104 104 185 185 185 F F F
119932024-02-03T17:27:49.506Z 24 480-499 100 100 100 181 181 181 T F T <---
119942024-02-03T17:27:49.506Z 25 500-519 106 106 106 196 196 196 F F F
119952024-02-03T17:27:49.506Z 26 520-539 100 100 100 181 181 181 T T T
119962024-02-03T17:27:49.507Z 27 540-559 106 106 106 192 192 192 T F T <---
119972024-02-03T17:27:49.507Z 28 560-579 108 108 108 200 200 200 F F F
119982024-02-03T17:27:49.507Z 29 580-599 106 106 106 197 197 197 F F F
119992024-02-03T17:27:49.507ZMax gen: 109, Max flush: 201
120002024-02-03T17:27:49.507ZError: Difference in extent metadata found!
120012024-02-03T17:27:49.510ZA Difference in extent metadata is expected here
120022024-02-03T17:27:49.511ZOn loop 55
120032024-02-03T17:27:49.511Z
120042024-02-03T17:27:49.511ZVerifying data now
120052024-02-03T17:27:55.838ZNo change after verify
120062024-02-03T17:27:55.838ZLoop: 55 Downstairs dump after verify (and repair):
120072024-02-03T17:27:55.845ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120082024-02-03T17:27:55.846Z 0 000-019 109 109 109 201 201 201 F F F
120092024-02-03T17:27:55.846Z 1 020-039 103 103 103 184 184 184 F F F
120102024-02-03T17:27:55.846Z 2 040-059 108 108 108 199 199 199 F F F
120112024-02-03T17:27:55.846Z 3 060-079 110 110 110 202 202 202 F F F
120122024-02-03T17:27:55.846Z 4 080-099 106 106 106 192 192 192 F F F
120132024-02-03T17:27:55.846Z 5 100-119 110 110 110 202 202 202 F F F
120142024-02-03T17:27:55.846Z 6 120-139 110 110 110 202 202 202 F F F
120152024-02-03T17:27:55.846Z 7 140-159 110 110 110 202 202 202 F F F
120162024-02-03T17:27:55.846Z 8 160-179 104 104 104 187 187 187 F F F
120172024-02-03T17:27:55.846Z 9 180-199 98 98 98 177 177 177 F F F
120182024-02-03T17:27:55.846Z 10 200-219 106 106 106 194 194 194 F F F
120192024-02-03T17:27:55.847Z 11 220-239 109 109 109 201 201 201 F F F
120202024-02-03T17:27:55.847Z 12 240-259 110 110 110 202 202 202 F F F
120212024-02-03T17:27:55.847Z 13 260-279 110 110 110 202 202 202 F F F
120222024-02-03T17:27:55.847Z 14 280-299 109 109 109 201 201 201 F F F
120232024-02-03T17:27:55.847Z 15 300-319 109 109 109 201 201 201 F F F
120242024-02-03T17:27:55.847Z 16 320-339 109 109 109 201 201 201 F F F
120252024-02-03T17:27:55.847Z 17 340-359 110 110 110 202 202 202 F F F
120262024-02-03T17:27:55.847Z 18 360-379 110 110 110 202 202 202 F F F
120272024-02-03T17:27:55.847Z 19 380-399 110 110 110 202 202 202 F F F
120282024-02-03T17:27:55.847Z 20 400-419 110 110 110 202 202 202 F F F
120292024-02-03T17:27:55.847Z 21 420-439 110 110 110 202 202 202 F F F
120302024-02-03T17:27:55.847Z 22 440-459 110 110 110 202 202 202 F F F
120312024-02-03T17:27:55.848Z 23 460-479 104 104 104 185 185 185 F F F
120322024-02-03T17:27:55.848Z 24 480-499 110 110 110 202 202 202 F F F
120332024-02-03T17:27:55.848Z 25 500-519 106 106 106 196 196 196 F F F
120342024-02-03T17:27:55.848Z 26 520-539 110 110 110 202 202 202 F F F
120352024-02-03T17:27:55.848Z 27 540-559 110 110 110 202 202 202 F F F
120362024-02-03T17:27:55.848Z 28 560-579 108 108 108 200 200 200 F F F
120372024-02-03T17:27:55.848Z 29 580-599 106 106 106 197 197 197 F F F
120382024-02-03T17:27:55.848ZMax gen: 110, Max flush: 202
120392024-02-03T17:27:55.848Z
120402024-02-03T17:27:55.848ZBegin loop 56
120412024-02-03T17:27:55.848ZDownstairs to restart: 0
120422024-02-03T17:27:55.878Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.04622247Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583}
120432024-02-03T17:27:55.878Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.046804495Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583}
120442024-02-03T17:27:55.884Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.046871726Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583}
120452024-02-03T17:27:55.884Z{Crucible runtime is spawned
120462024-02-03T17:27:55.884Z"msg":"Crucible 988ce6bb-9ea5-4908-8c10-d201f3f2454b has session id: 0e111e6e-52dd-4638-97dd-1eb1381d6e9b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.046957207Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120472024-02-03T17:27:55.884Z{"msg":"Upstairs opts: Upstairs UUID: 988ce6bb-9ea5-4908-8c10-d201f3f2454b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047130713Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120482024-02-03T17:27:55.884Z{"msg":"Crucible stats registered with UUID: 988ce6bb-9ea5-4908-8c10-d201f3f2454b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.04719232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120492024-02-03T17:27:55.885Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047235122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583}
120502024-02-03T17:27:55.885Z{"msg":"988ce6bb-9ea5-4908-8c10-d201f3f2454b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047286388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120512024-02-03T17:27:55.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047327181Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120522024-02-03T17:27:55.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047368662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120532024-02-03T17:27:55.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047415046Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120542024-02-03T17:27:55.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047458347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120552024-02-03T17:27:55.885Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047498904Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120562024-02-03T17:27:55.885Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047524221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120572024-02-03T17:27:55.885Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047581859Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120582024-02-03T17:27:55.885Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047610889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120592024-02-03T17:27:55.885Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047651095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120602024-02-03T17:27:55.885Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.047989613Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120612024-02-03T17:27:55.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.048154899Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120622024-02-03T17:27:55.886Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.048201624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120632024-02-03T17:27:55.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.049524488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120642024-02-03T17:27:55.886Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.04959116Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120652024-02-03T17:27:55.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.049617557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120662024-02-03T17:27:55.886Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.049640002Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120672024-02-03T17:27:55.886Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.050362347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120682024-02-03T17:27:55.886Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.050438888Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120692024-02-03T17:27:55.886Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.050486832Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120702024-02-03T17:27:55.886Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.05093712Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120712024-02-03T17:27:55.893Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.051463725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120722024-02-03T17:27:55.893Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.051550228Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120732024-02-03T17:27:55.893Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.051599276Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120742024-02-03T17:27:55.893Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.051640843Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120752024-02-03T17:27:55.894Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.051788241Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120762024-02-03T17:27:55.894Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.051842955Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120772024-02-03T17:27:55.894Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.051869266Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120782024-02-03T17:27:55.894Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052358907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120792024-02-03T17:27:55.894Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052467355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120802024-02-03T17:27:55.894Z{"msg":"[0]R flush_numbers[0..12]: [201, 184, 199, 202, 192, 202, 202, 202, 187, 177, 194, 201]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.0525146Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120812024-02-03T17:27:55.894Z{"msg":"[0]R generation[0..12]: [109, 103, 108, 110, 106, 110, 110, 110, 104, 98, 106, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.05257956Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120822024-02-03T17:27:55.894Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052608291Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120832024-02-03T17:27:55.894Z{"msg":"[1]R flush_numbers[0..12]: [201, 184, 199, 202, 192, 202, 202, 202, 187, 177, 194, 201]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
120842024-02-03T17:27:55.894Z,"time":"2024-02-03T17:27:55.052651833Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120852024-02-03T17:27:55.894Z{"msg":"[1]R generation[0..12]: [109, 103, 108, 110, 106, 110, 110, 110, 104, 98, 106, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052678017Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120862024-02-03T17:27:55.894Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052701527Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120872024-02-03T17:27:55.894Z{"msg":"[2]R flush_numbers[0..12]: [201, 184, 199, 202, 192, 202, 202, 202, 187, 177, 194, 201]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052725069Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120882024-02-03T17:27:55.895Z{"msg":"[2]R generation[0..12]: [109, 103, 108, 110, 106, 110, 110, 110, 104, 98, 106, 109]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052774724Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120892024-02-03T17:27:55.895Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052799895Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120902024-02-03T17:27:55.895Z{"msg":"Max found gen is 111","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.05284005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120912024-02-03T17:27:55.895Z{"msg":"Generation requested: 112 >= found:111","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052864606Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120922024-02-03T17:27:55.895Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052906583Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120932024-02-03T17:27:55.895Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052929983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120942024-02-03T17:27:55.895Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052974601Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120952024-02-03T17:27:55.895Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.052996674Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120962024-02-03T17:27:55.895Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.053040738Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120972024-02-03T17:27:55.895Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.053105478Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120982024-02-03T17:27:55.895Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.053130464Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
120992024-02-03T17:27:55.895Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.053173945Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
121002024-02-03T17:27:55.896Z{"msg":"988ce6bb-9ea5-4908-8c10-d201f3f2454b is now active with session: 0e111e6e-52dd-4638-97dd-1eb1381d6e9b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.053197412Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
121012024-02-03T17:27:55.896Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
121022024-02-03T17:27:55.896Zlevel":30,"time":"2024-02-03T17:27:55.053240735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583,"session_id":"0e111e6e-52dd-4638-97dd-1eb1381d6e9b"}
121032024-02-03T17:27:55.896Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:27:55.053263516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583}
121042024-02-03T17:27:55.896ZLoading write count information from file "/tmp/test_repair_verify.data"
121052024-02-03T17:27:55.896ZRun Repair workload
121062024-02-03T17:27:55.896Z01/30 Write block 361 len 20480 data: 15 13 15 13 10
121072024-02-03T17:27:55.896Z02/30 Read block 51 len 20480
121082024-02-03T17:27:55.896Z03/30 Read block 81 len 12288
121092024-02-03T17:27:55.900Z04/30 Write block 515 len 28672 data: 7 5 6 5 4 3 3
121102024-02-03T17:27:55.901Z05/30 Read block 92 len 36864
121112024-02-03T17:27:55.901Z06/30 Read block 529 len 24576
121122024-02-03T17:27:55.901Z07/30 Read block 155 len 20480
121132024-02-03T17:27:55.901Z08/30 Read block 188 len 20480
121142024-02-03T17:27:55.901Z09/30 Write block 280 len 12288 data: 10 12 13
121152024-02-03T17:27:55.901Z10/30 Read block 62 len 40960
121162024-02-03T17:27:55.905Z11/30 Read block 574 len 28672
121172024-02-03T17:27:55.905Z12/30 Flush
121182024-02-03T17:27:55.908Z13/30 Write block 575 len 28672 data: 7 9 10 12 12 12 11
121192024-02-03T17:27:55.908Z14/30 Write block 89 len 36864 data: 8 9 8 9 9 10 8 7 8
121202024-02-03T17:27:55.908Z15/30 Flush
121212024-02-03T17:27:55.922Z16/30 Write block 496 len 8192 data: 6 7
121222024-02-03T17:27:55.925Z17/30 Write block 134 len 28672 data: 13 15 16 17 18 16 12
121232024-02-03T17:27:55.925Z18/30 Read block 138 len 40960
121242024-02-03T17:27:55.925Z19/30 Read block 95 len 12288
121252024-02-03T17:27:55.926Z20/30 Write block 275 len 4096 data: 15
121262024-02-03T17:27:55.926Z21/30 Read block 434 len 24576
121272024-02-03T17:27:55.926Z22/30 Write block 60 len 24576 data: 11 10 9 11 11 8
121282024-02-03T17:27:55.926Z23/30 Write block 290 len 32768 data: 7 8 8 10 10 8 9 10
121292024-02-03T17:27:55.926Z24/30 Write block 308 len 40960 data: 11 11 9 8 5 7 7 7 7 6
121302024-02-03T17:27:55.926Z25/30 Write block 227 len 8192 data: 17 14
121312024-02-03T17:27:55.930Z26/30 Flush
121322024-02-03T17:27:55.973Z27/30 Write block 407 len 16384 data: 12 9 10 9
121332024-02-03T17:27:55.984Z28/30 Read block 585 len 20480
121342024-02-03T17:27:55.984Z29/30 Write block 265 len 4096 data: 10
121352024-02-03T17:27:55.984Z30/30 Read block 349 len 8192
121362024-02-03T17:27:55.984Z----------------------------------------------------------------
121372024-02-03T17:27:55.984Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
121382024-02-03T17:27:55.984ZStates: Active Active Active
121392024-02-03T17:27:55.984ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121402024-02-03T17:27:55.984Z 1 Acked 1000 Write 5 Sent Done Done false
121412024-02-03T17:27:55.984Z 2 Acked 1001 Read 5 Sent Done Done false
121422024-02-03T17:27:55.984Z 3 Acked 1002 Read 3 Sent Done Done false
121432024-02-03T17:27:55.985Z 4 Acked 1003 Write 7 Sent Done Done false
121442024-02-03T17:27:55.985Z 5 Acked 1004 Read 9 Sent Done Done false
121452024-02-03T17:27:55.985Z 6 Acked 1005 Read 6 Sent Done Done false
121462024-02-03T17:27:55.985Z 7 Acked 1006 Read 5 Sent Done Done false
121472024-02-03T17:27:55.985Z 8 Acked 1007 Read 5 Sent Done Done false
121482024-02-03T17:27:55.985Z 9 Acked 1008 Write 3 Sent Done Done false
121492024-02-03T17:27:55.985Z 10 Acked 1009 Read 10 Sent Done Done false
121502024-02-03T17:27:55.985Z 11 Acked 1010 Read 7 Sent Done Done false
121512024-02-03T17:27:55.985Z 12 Acked 1011 Flush 0 Sent Done Done false
121522024-02-03T17:27:55.985Z 13 Acked 1012 Write 7 Sent Done Done false
121532024-02-03T17:27:55.985Z 14 Acked 1013 Write 9 Sent Done Done false
121542024-02-03T17:27:55.985Z 15 Acked 1014 Flush 0 Sent Done Done false
121552024-02-03T17:27:55.985Z 16 Acked 1015 Write 2 Sent Done Done false
121562024-02-03T17:27:55.985Z 17 Acked 1016 Write 7 Sent Done Done false
121572024-02-03T17:27:55.986Z 18 Acked 1017 Read 10 Sent Done Done false
121582024-02-03T17:27:55.986Z 19 Acked 1018 Read 3 Sent Done Done false
121592024-02-03T17:27:55.986Z 20 Acked 1019 Write 1 Sent Done Done false
121602024-02-03T17:27:55.986Z 21 Acked 1020 Read 6 Sent Done Done false
121612024-02-03T17:27:55.986Z 22 Acked 1021 Write 6 Sent Done Done false
121622024-02-03T17:27:55.986Z 23 Acked 1022 Write 8 Sent Done Done false
121632024-02-03T17:27:55.986Z 24 Acked 1023 Write 10 Sent Done Done false
121642024-02-03T17:27:55.986Z 25 Acked 1024 Write 2 Sent Done Done false
121652024-02-03T17:27:55.986Z 26 Acked 1025 Flush 0 Sent Done Done false
121662024-02-03T17:27:55.986Z 27 Acked 1026 Write 4 Sent Done Done false
121672024-02-03T17:27:55.986Z 28 Acked 1027 Read 5 Sent Done Done false
121682024-02-03T17:27:55.986Z 29 Acked 1028 Write 1 Sent Done Done false
121692024-02-03T17:27:55.986Z 30 Acked 1029 Read 2 Sent Sent Done false
121702024-02-03T17:27:55.987Z STATES DS:0 DS:1 DS:2 TOTAL
121712024-02-03T17:27:55.987Z New 0 0 0 0
121722024-02-03T17:27:55.987Z Sent 30 1 0 31
121732024-02-03T17:27:55.987Z Done 0 29 30 59
121742024-02-03T17:27:55.987Z Skipped 0 0 0 0
121752024-02-03T17:27:55.987Z Error 0 0 0 0
121762024-02-03T17:27:55.987ZLast Flush: 0 1025 1025
121772024-02-03T17:27:55.987ZDownstairs last five completed:
121782024-02-03T17:27:55.987ZUpstairs last five completed: 30 29 28 27 26
121792024-02-03T17:27:55.987Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:27:55.144906661Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1583}
121802024-02-03T17:27:55.987ZWrote out file "/tmp/test_repair_verify.data"
121812024-02-03T17:27:55.994Z
121822024-02-03T17:27:55.994ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121832024-02-03T17:27:56.019ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121842024-02-03T17:27:56.020Z 0 000-019 109 109 109 201 201 201 F F F
121852024-02-03T17:27:56.020Z 1 020-039 103 103 103 184 184 184 F F F
121862024-02-03T17:27:56.020Z 2 040-059 108 108 108 199 199 199 F F F
121872024-02-03T17:27:56.025Z 3 060-079 110 112 112 202 205 205 F F F <---
121882024-02-03T17:27:56.026Z 4 080-099 106 112 112 192 204 204 F F F <---
121892024-02-03T17:27:56.026Z 5 100-119 110 110 110 202 202 202 F F F
121902024-02-03T17:27:56.026Z 6 120-139 110 112 112 202 205 205 F F F <---
121912024-02-03T17:27:56.026Z 7 140-159 110 112 112 202 205 205 F F F <---
121922024-02-03T17:27:56.026Z 8 160-179 104 104 104 187 187 187 F F F
121932024-02-03T17:27:56.026Z 9 180-199 98 98 98 177 177 177 F F F
121942024-02-03T17:27:56.026Z 10 200-219 106 106 106 194 194 194 F F F
121952024-02-03T17:27:56.026Z 11 220-239 109 112 112 201 205 205 F F F <---
121962024-02-03T17:27:56.026Z 12 240-259 110 110 110 202 202 202 F F F
121972024-02-03T17:27:56.026Z 13 260-279 110 112 112 202 205 205 F T T <---
121982024-02-03T17:27:56.026Z 14 280-299 109 112 112 201 205 205 F F F <---
121992024-02-03T17:27:56.026Z 15 300-319 109 112 112 201 205 205 F F F <---
122002024-02-03T17:27:56.026Z 16 320-339 109 109 109 201 201 201 F F F
122012024-02-03T17:27:56.026Z 17 340-359 110 110 110 202 202 202 F F F
122022024-02-03T17:27:56.026Z 18 360-379 110 112 112 202 203 203 F F F <---
122032024-02-03T17:27:56.026Z 19 380-399 110 110 110 202 202 202 F F F
122042024-02-03T17:27:56.026Z 20 400-419 110 110 110 202 202 202 F T T <---
122052024-02-03T17:27:56.026Z 21 420-439 110 110 110 202 202 202 F F F
122062024-02-03T17:27:56.026Z 22 440-459 110 110 110 202 202 202 F F F
122072024-02-03T17:27:56.027Z 23 460-479 104 104 104 185 185 185 F F F
122082024-02-03T17:27:56.027Z 24 480-499 110 112 112 202 205 205 F F F <---
122092024-02-03T17:27:56.027Z 25 500-519 106 112 112 196 203 203 F F F <---
122102024-02-03T17:27:56.027Z 26 520-539 110 112 112 202 203 203 F F F <---
122112024-02-03T17:27:56.027Z 27 540-559 110 110 110 202 202 202 F F F
122122024-02-03T17:27:56.027Z 28 560-579 108 112 112 200 204 204 F F F <---
122132024-02-03T17:27:56.027Z 29 580-599 106 112 112 197 204 204 F F F <---
122142024-02-03T17:27:56.027ZMax gen: 112, Max flush: 205
122152024-02-03T17:27:56.027ZError: Difference in extent metadata found!
122162024-02-03T17:27:56.027ZA Difference in extent metadata is expected here
122172024-02-03T17:27:56.027ZOn loop 56
122182024-02-03T17:27:56.027Z
122192024-02-03T17:27:56.032ZVerifying data now
122202024-02-03T17:28:01.296ZNo change after verify
122212024-02-03T17:28:01.296ZLoop: 56 Downstairs dump after verify (and repair):
122222024-02-03T17:28:01.321ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122232024-02-03T17:28:01.321Z 0 000-019 109 109 109 201 201 201 F F F
122242024-02-03T17:28:01.324Z 1 020-039 103 103 103 184 184 184 F F F
122252024-02-03T17:28:01.325Z 2 040-059 108 108 108 199 199 199 F F F
122262024-02-03T17:28:01.325Z 3 060-079 112 112 112 205 205 205 F F F
122272024-02-03T17:28:01.325Z 4 080-099 112 112 112 204 204 204 F F F
122282024-02-03T17:28:01.325Z 5 100-119 110 110 110 202 202 202 F F F
122292024-02-03T17:28:01.325Z 6 120-139 112 112 112 205 205 205 F F F
122302024-02-03T17:28:01.325Z 7 140-159 112 112 112 205 205 205 F F F
122312024-02-03T17:28:01.325Z 8 160-179 104 104 104 187 187 187 F F F
122322024-02-03T17:28:01.325Z 9 180-199 98 98 98 177 177 177 F F F
122332024-02-03T17:28:01.325Z 10 200-219 106 106 106 194 194 194 F F F
122342024-02-03T17:28:01.325Z 11 220-239 112 112 112 205 205 205 F F F
122352024-02-03T17:28:01.325Z 12 240-259 110 110 110 202 202 202 F F F
122362024-02-03T17:28:01.325Z 13 260-279 113 113 113 206 206 206 F F F
122372024-02-03T17:28:01.325Z 14 280-299 112 112 112 205 205 205 F F F
122382024-02-03T17:28:01.325Z 15 300-319 112 112 112 205 205 205 F F F
122392024-02-03T17:28:01.325Z 16 320-339 109 109 109 201 201 201 F F F
122402024-02-03T17:28:01.325Z 17 340-359 110 110 110 202 202 202 F F F
122412024-02-03T17:28:01.325Z 18 360-379 112 112 112 203 203 203 F F F
122422024-02-03T17:28:01.325Z 19 380-399 110 110 110 202 202 202 F F F
122432024-02-03T17:28:01.325Z 20 400-419 113 113 113 206 206 206 F F F
122442024-02-03T17:28:01.325Z 21 420-439 110 110 110 202 202 202 F F F
122452024-02-03T17:28:01.325Z 22 440-459 110 110 110 202 202 202 F F F
122462024-02-03T17:28:01.325Z 23 460-479 104 104 104 185 185 185 F F F
122472024-02-03T17:28:01.325Z 24 480-499 112 112 112 205 205 205 F F F
122482024-02-03T17:28:01.326Z 25 500-519 112 112 112 203 203 203 F F F
122492024-02-03T17:28:01.326Z 26 520-539 112 112 112 203 203 203 F F F
122502024-02-03T17:28:01.326Z 27 540-559 110 110 110 202 202 202 F F F
122512024-02-03T17:28:01.326Z 28 560-579 112 112 112 204 204 204 F F F
122522024-02-03T17:28:01.326Z 29 580-599 112 112 112 204 204 204 F F F
122532024-02-03T17:28:01.326ZMax gen: 113, Max flush: 206
122542024-02-03T17:28:01.326Z
122552024-02-03T17:28:01.326ZBegin loop 57
122562024-02-03T17:28:01.326ZDownstairs to restart: 1
122572024-02-03T17:28:01.358Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.525712948Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591}
122582024-02-03T17:28:01.364Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.526197689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591}
122592024-02-03T17:28:01.364Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.526277351Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591}
122602024-02-03T17:28:01.364Z{"msg":"Crucible 730d3b6a-250c-4fee-8301-40f0528aa9b7 has session id: 8968e184-9aa4-4c81-be6e-eb538c405492","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.526398367Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122612024-02-03T17:28:01.364Z{"msg":"Upstairs opts: Upstairs UUID: 730d3b6a-250c-4fee-8301-40f0528aa9b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.526519359Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122622024-02-03T17:28:01.364Z{"msg":"Crucible stats registered with UUID: 730d3b6a-250c-4fee-8301-40f0528aa9b7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.526564199Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122632024-02-03T17:28:01.364ZCrucible runtime is spawned
122642024-02-03T17:28:01.364Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.526836931Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591}
122652024-02-03T17:28:01.364Z{"msg":"730d3b6a-250c-4fee-8301-40f0528aa9b7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.526989176Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122662024-02-03T17:28:01.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527031276Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122672024-02-03T17:28:01.364Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527124005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122682024-02-03T17:28:01.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527167558Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122692024-02-03T17:28:01.364Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527209262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122702024-02-03T17:28:01.364Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527249504Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122712024-02-03T17:28:01.364Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.52732234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122722024-02-03T17:28:01.364Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527369346Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122732024-02-03T17:28:01.364Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527541853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122742024-02-03T17:28:01.364Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527715256Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122752024-02-03T17:28:01.364Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527841958Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122762024-02-03T17:28:01.364Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.527941883Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122772024-02-03T17:28:01.364Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.528043111Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122782024-02-03T17:28:01.364Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.528853337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122792024-02-03T17:28:01.364Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.529000232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122802024-02-03T17:28:01.364Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.529232883Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122812024-02-03T17:28:01.365Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.529284262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122822024-02-03T17:28:01.365Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.529331503Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122832024-02-03T17:28:01.365Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.52937989Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122842024-02-03T17:28:01.365Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.5300381Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122852024-02-03T17:28:01.365Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.530192658Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122862024-02-03T17:28:01.365Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.530262944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122872024-02-03T17:28:01.370Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531198597Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122882024-02-03T17:28:01.370Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531292878Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122892024-02-03T17:28:01.370Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531324996Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122902024-02-03T17:28:01.370Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531408715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122912024-02-03T17:28:01.370Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.53146879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122922024-02-03T17:28:01.370Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531495448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122932024-02-03T17:28:01.370Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531536234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122942024-02-03T17:28:01.370Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.53156132Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122952024-02-03T17:28:01.370Z{"msg":"[0]R flush_numbers[0..12]: [201, 184, 199, 205, 204, 202, 205, 205, 187, 177, 194, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531604758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122962024-02-03T17:28:01.370Z{"msg":"[0]R generation[0..12]: [109, 103, 108, 112, 112, 110, 112, 112, 104, 98, 106, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531630197Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122972024-02-03T17:28:01.370Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531654248Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122982024-02-03T17:28:01.370Z{"msg":"[1]R flush_numbers[0..12]: [201, 184, 199, 205, 204, 202, 205, 205, 187, 177, 194, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531698902Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
122992024-02-03T17:28:01.370Z{"msg":"Wait for a query_work_queue command to finish before sending IO
123002024-02-03T17:28:01.370Z[1]R generation[0..12]: [109, 103, 108, 112, 112, 110, 112, 112, 104, 98, 106, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531725801Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123012024-02-03T17:28:01.370Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531782967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123022024-02-03T17:28:01.371Z{"msg":"[2]R flush_numbers[0..12]: [201, 184, 199, 205, 204, 202, 205, 205, 187, 177, 194, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.53180838Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123032024-02-03T17:28:01.371Z{"msg":"[2]R generation[0..12]: [109, 103, 108, 112, 112, 110, 112, 112, 104, 98, 106, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531832975Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123042024-02-03T17:28:01.371Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531875366Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123052024-02-03T17:28:01.371Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531899302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123062024-02-03T17:28:01.371Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.531946487Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123072024-02-03T17:28:01.371Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532017531Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123082024-02-03T17:28:01.371Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532116221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123092024-02-03T17:28:01.371Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532153629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123102024-02-03T17:28:01.371Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532221733Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123112024-02-03T17:28:01.371Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532254271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123122024-02-03T17:28:01.371Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123132024-02-03T17:28:01.371Z,"time":"2024-02-03T17:28:00.532324609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123142024-02-03T17:28:01.371Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532354471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123152024-02-03T17:28:01.371Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532416012Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123162024-02-03T17:28:01.371Z{"msg":"730d3b6a-250c-4fee-8301-40f0528aa9b7 is now active with session: 8968e184-9aa4-4c81-be6e-eb538c405492","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532443691Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123172024-02-03T17:28:01.371Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.5324833Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591,"session_id":"8968e184-9aa4-4c81-be6e-eb538c405492"}
123182024-02-03T17:28:01.371Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:00.532505671Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591}
123192024-02-03T17:28:01.371ZLoading write count information from file "/tmp/test_repair_verify.data"
123202024-02-03T17:28:01.371ZRun Repair workload
123212024-02-03T17:28:01.371Z01/30 Write block 164 len 12288 data: 9 8 7
123222024-02-03T17:28:01.371Z02/30 Read block 458 len 28672
123232024-02-03T17:28:01.371Z03/30 Read block 34 len 20480
123242024-02-03T17:28:01.375Z04/30 Read block 565 len 36864
123252024-02-03T17:28:01.375Z05/30 Read block 417 len 32768
123262024-02-03T17:28:01.375Z06/30 Write block 5 len 12288 data: 8 8 10
123272024-02-03T17:28:01.375Z07/30 Read block 262 len 40960
123282024-02-03T17:28:01.378Z08/30 Read block 493 len 8192
123292024-02-03T17:28:01.378Z09/30 Read block 407 len 8192
123302024-02-03T17:28:01.378Z10/30 Write block 421 len 32768 data: 12 11 11 10 8 6 6 5
123312024-02-03T17:28:01.381Z11/30 Write block 76 len 4096 data: 7
123322024-02-03T17:28:01.381Z12/30 Write block 240 len 36864 data: 8 8 5 4 4 5 6 7 8
123332024-02-03T17:28:01.384Z13/30 Write block 549 len 24576 data: 12 11 10 8 8 8
123342024-02-03T17:28:01.384Z14/30 Read block 98 len 12288
123352024-02-03T17:28:01.387Z15/30 Read block 16 len 12288
123362024-02-03T17:28:01.387Z16/30 Write block 258 len 20480 data: 5 4 6 8 10
123372024-02-03T17:28:01.387Z17/30 Read block 469 len 4096
123382024-02-03T17:28:01.390Z18/30 Write block 380 len 12288 data: 10 10 8
123392024-02-03T17:28:01.390Z19/30 Write block 430 len 32768 data: 6 6 6 4 6 7 7 8
123402024-02-03T17:28:01.390Z20/30 Read block 457 len 24576
123412024-02-03T17:28:01.394Z21/30 Read block 243 len 16384
123422024-02-03T17:28:01.394Z22/30 Read block 335 len 32768
123432024-02-03T17:28:01.399Z23/30 Read block 97 len 28672
123442024-02-03T17:28:01.399Z24/30 Read block 298 len 24576
123452024-02-03T17:28:01.399Z25/30 Read block 339 len 12288
123462024-02-03T17:28:01.402Z26/30 Write block 194 len 12288 data: 4 5 7
123472024-02-03T17:28:01.402Z27/30 Write block 589 len 20480 data: 10 11 11 9 9
123482024-02-03T17:28:01.402Z28/30 Write block 518 len 20480 data: 6 5 4 4 5
123492024-02-03T17:28:01.405Z29/30 Read block 194 len 20480
123502024-02-03T17:28:01.405Z30/30 Read block 50 len 12288
123512024-02-03T17:28:01.409Z----------------------------------------------------------------
123522024-02-03T17:28:01.409Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123532024-02-03T17:28:01.409ZStates: Active Active Active
123542024-02-03T17:28:01.409ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123552024-02-03T17:28:01.409Z 1 Acked 1000 Write 3 Done Sent Done false
123562024-02-03T17:28:01.409Z 2 Acked 1001 Read 7 Done Sent Done false
123572024-02-03T17:28:01.409Z 3 Acked 1002 Read 5 Done Sent Done false
123582024-02-03T17:28:01.409Z 4 Acked 1003 Read 9 Done Sent Done false
123592024-02-03T17:28:01.409Z 5 Acked 1004 Read 8 Done Sent Done false
123602024-02-03T17:28:01.409Z 6 Acked 1005 Write 3 Done Sent Done false
123612024-02-03T17:28:01.409Z 7 Acked 1006 Read 10 Done Sent Done false
123622024-02-03T17:28:01.409Z 8 Acked 1007 Read 2 Done Sent Done false
123632024-02-03T17:28:01.409Z 9 Acked 1008 Read 2 Done Sent Done false
123642024-02-03T17:28:01.409Z 10 Acked 1009 Write 8 Done Sent Done false
123652024-02-03T17:28:01.409Z 11 Acked 1010 Write 1 Done Sent Done false
123662024-02-03T17:28:01.409Z 12 Acked 1011 Write 9 Done Sent Done false
123672024-02-03T17:28:01.409Z 13 Acked 1012 Write 6 Done Sent Done false
123682024-02-03T17:28:01.410Z 14 Acked 1013 Read 3 Done Sent Done false
123692024-02-03T17:28:01.410Z 15 Acked 1014 Read 3 Done Sent Done false
123702024-02-03T17:28:01.410Z 16 Acked 1015 Write 5 Done Sent Done false
123712024-02-03T17:28:01.410Z 17 Acked 1016 Read 1 Done Sent Done false
123722024-02-03T17:28:01.410Z 18 Acked 1017 Write 3 Done Sent Done false
123732024-02-03T17:28:01.410Z 19 Acked 1018 Write 8 Done Sent Done false
123742024-02-03T17:28:01.410Z 20 Acked 1019 Read 6 Done Sent Done false
123752024-02-03T17:28:01.410Z 21 Acked 1020 Read 4 Done Sent Done false
123762024-02-03T17:28:01.410Z 22 Acked 1021 Read 8 Done Sent Done false
123772024-02-03T17:28:01.410Z 23 Acked 1022 Read 7 Done Sent Done false
123782024-02-03T17:28:01.410Z 24 Acked 1023 Read 6 Done Sent Done false
123792024-02-03T17:28:01.410Z 25 Acked 1024 Read 3 Done Sent Done false
123802024-02-03T17:28:01.410Z 26 Acked 1025 Write 3 Done Sent Done false
123812024-02-03T17:28:01.410Z 27 Acked 1026 Write 5 Done Sent Done false
123822024-02-03T17:28:01.410Z 28 Acked 1027 Write 5 Done Sent Done false
123832024-02-03T17:28:01.410Z 29 Acked 1028 Read 5 Done Sent Done false
123842024-02-03T17:28:01.410Z 30 Acked 1029 Read 3 Done Sent Done false
123852024-02-03T17:28:01.410Z STATES DS:0 DS:1 DS:2 TOTAL
123862024-02-03T17:28:01.410Z New 0 0 0 0
123872024-02-03T17:28:01.410Z Sent 0 30 0 30
123882024-02-03T17:28:01.410Z Done 30 0 30 60
123892024-02-03T17:28:01.410Z Skipped 0 0 0 0
123902024-02-03T17:28:01.410Z Error 0 0 0 0
123912024-02-03T17:28:01.410ZLast Flush: 0 0 0
123922024-02-03T17:28:01.411ZDownstairs last five completed:
123932024-02-03T17:28:01.411ZUpstairs last five completed: 30 29 28 27 26
123942024-02-03T17:28:01.411Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:00.575933679Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1591}
123952024-02-03T17:28:01.411ZWrote out file "/tmp/test_repair_verify.data"
123962024-02-03T17:28:01.414Z
123972024-02-03T17:28:01.424ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123982024-02-03T17:28:01.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123992024-02-03T17:28:01.453Z 0 000-019 109 109 109 201 201 201 T F T <---
124002024-02-03T17:28:01.453Z 1 020-039 103 103 103 184 184 184 F F F
124012024-02-03T17:28:01.453Z 2 040-059 108 108 108 199 199 199 F F F
124022024-02-03T17:28:01.453Z 3 060-079 112 112 112 205 205 205 T F T <---
124032024-02-03T17:28:01.457Z 4 080-099 112 112 112 204 204 204 F F F
124042024-02-03T17:28:01.457Z 5 100-119 110 110 110 202 202 202 F F F
124052024-02-03T17:28:01.457Z 6 120-139 112 112 112 205 205 205 F F F
124062024-02-03T17:28:01.457Z 7 140-159 112 112 112 205 205 205 F F F
124072024-02-03T17:28:01.457Z 8 160-179 104 104 104 187 187 187 T F T <---
124082024-02-03T17:28:01.457Z 9 180-199 98 98 98 177 177 177 T F T <---
124092024-02-03T17:28:01.457Z 10 200-219 106 106 106 194 194 194 F F F
124102024-02-03T17:28:01.457Z 11 220-239 112 112 112 205 205 205 F F F
124112024-02-03T17:28:01.457Z 12 240-259 110 110 110 202 202 202 T F T <---
124122024-02-03T17:28:01.457Z 13 260-279 113 113 113 206 206 206 T F T <---
124132024-02-03T17:28:01.457Z 14 280-299 112 112 112 205 205 205 F F F
124142024-02-03T17:28:01.457Z 15 300-319 112 112 112 205 205 205 F F F
124152024-02-03T17:28:01.458Z 16 320-339 109 109 109 201 201 201 F F F
124162024-02-03T17:28:01.458Z 17 340-359 110 110 110 202 202 202 F F F
124172024-02-03T17:28:01.458Z 18 360-379 112 112 112 203 203 203 F F F
124182024-02-03T17:28:01.458Z 19 380-399 110 110 110 202 202 202 T F T <---
124192024-02-03T17:28:01.458Z 20 400-419 113 113 113 206 206 206 F F F
124202024-02-03T17:28:01.458Z 21 420-439 110 110 110 202 202 202 T F T <---
124212024-02-03T17:28:01.458Z 22 440-459 110 110 110 202 202 202 F F F
124222024-02-03T17:28:01.458Z 23 460-479 104 104 104 185 185 185 F F F
124232024-02-03T17:28:01.458Z 24 480-499 112 112 112 205 205 205 F F F
124242024-02-03T17:28:01.458Z 25 500-519 112 112 112 203 203 203 T F T <---
124252024-02-03T17:28:01.458Z 26 520-539 112 112 112 203 203 203 T F T <---
124262024-02-03T17:28:01.458Z 27 540-559 110 110 110 202 202 202 T F T <---
124272024-02-03T17:28:01.458Z 28 560-579 112 112 112 204 204 204 F F F
124282024-02-03T17:28:01.458Z 29 580-599 112 112 112 204 204 204 T F T <---
124292024-02-03T17:28:01.458ZMax gen: 113, Max flush: 206
124302024-02-03T17:28:01.458ZError: Difference in extent metadata found!
124312024-02-03T17:28:01.462ZA Difference in extent metadata is expected here
124322024-02-03T17:28:01.462ZOn loop 57
124332024-02-03T17:28:01.462Z
124342024-02-03T17:28:01.465ZVerifying data now
124352024-02-03T17:28:06.582ZNo change after verify
124362024-02-03T17:28:06.582ZLoop: 57 Downstairs dump after verify (and repair):
124372024-02-03T17:28:06.606ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124382024-02-03T17:28:06.618Z 0 000-019 114 114 114 207 207 207 F F F
124392024-02-03T17:28:06.618Z 1 020-039 103 103 103 184 184 184 F F F
124402024-02-03T17:28:06.618Z 2 040-059 108 108 108 199 199 199 F F F
124412024-02-03T17:28:06.618Z 3 060-079 114 114 114 207 207 207 F F F
124422024-02-03T17:28:06.618Z 4 080-099 112 112 112 204 204 204 F F F
124432024-02-03T17:28:06.619Z 5 100-119 110 110 110 202 202 202 F F F
124442024-02-03T17:28:06.619Z 6 120-139 112 112 112 205 205 205 F F F
124452024-02-03T17:28:06.619Z 7 140-159 112 112 112 205 205 205 F F F
124462024-02-03T17:28:06.619Z 8 160-179 114 114 114 207 207 207 F F F
124472024-02-03T17:28:06.619Z 9 180-199 114 114 114 207 207 207 F F F
124482024-02-03T17:28:06.619Z 10 200-219 106 106 106 194 194 194 F F F
124492024-02-03T17:28:06.619Z 11 220-239 112 112 112 205 205 205 F F F
124502024-02-03T17:28:06.620Z 12 240-259 114 114 114 207 207 207 F F F
124512024-02-03T17:28:06.620Z 13 260-279 114 114 114 207 207 207 F F F
124522024-02-03T17:28:06.620Z 14 280-299 112 112 112 205 205 205 F F F
124532024-02-03T17:28:06.620Z 15 300-319 112 112 112 205 205 205 F F F
124542024-02-03T17:28:06.620Z 16 320-339 109 109 109 201 201 201 F F F
124552024-02-03T17:28:06.620Z 17 340-359 110 110 110 202 202 202 F F F
124562024-02-03T17:28:06.620Z 18 360-379 112 112 112 203 203 203 F F F
124572024-02-03T17:28:06.621Z 19 380-399 114 114 114 207 207 207 F F F
124582024-02-03T17:28:06.621Z 20 400-419 113 113 113 206 206 206 F F F
124592024-02-03T17:28:06.621Z 21 420-439 114 114 114 207 207 207 F F F
124602024-02-03T17:28:06.621Z 22 440-459 110 110 110 202 202 202 F F F
124612024-02-03T17:28:06.621Z 23 460-479 104 104 104 185 185 185 F F F
124622024-02-03T17:28:06.621Z 24 480-499 112 112 112 205 205 205 F F F
124632024-02-03T17:28:06.621Z 25 500-519 114 114 114 207 207 207 F F F
124642024-02-03T17:28:06.621Z 26 520-539 114 114 114 207 207 207 F F F
124652024-02-03T17:28:06.621Z 27 540-559 114 114 114 207 207 207 F F F
124662024-02-03T17:28:06.622Z 28 560-579 112 112 112 204 204 204 F F F
124672024-02-03T17:28:06.622Z 29 580-599 114 114 114 207 207 207 F F F
124682024-02-03T17:28:06.622ZMax gen: 114, Max flush: 207
124692024-02-03T17:28:06.630Z
124702024-02-03T17:28:06.630ZBegin loop 58
124712024-02-03T17:28:06.630ZDownstairs to restart: 0
124722024-02-03T17:28:06.646Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.813503258Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599}
124732024-02-03T17:28:06.646Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.813963838Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599}
124742024-02-03T17:28:06.651Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814006949Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal"Crucible runtime is spawned
124752024-02-03T17:28:06.651Z,"pid":1599}
124762024-02-03T17:28:06.652Z{"msg":"Crucible 46296428-9219-442d-805f-049e9b0a2534 has session id: 19e3d758-42ed-451a-b2db-674cef36797b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814309886Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124772024-02-03T17:28:06.652Z{"msg":"Upstairs opts: Upstairs UUID: 46296428-9219-442d-805f-049e9b0a2534, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814389795Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124782024-02-03T17:28:06.652Z{"msg":"Crucible stats registered with UUID: 46296428-9219-442d-805f-049e9b0a2534","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814420796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124792024-02-03T17:28:06.652Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814461679Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599}
124802024-02-03T17:28:06.652Z{"msg":"46296428-9219-442d-805f-049e9b0a2534 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814482375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124812024-02-03T17:28:06.652Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814583085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124822024-02-03T17:28:06.652Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814626219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124832024-02-03T17:28:06.652Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814667351Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124842024-02-03T17:28:06.652Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814708878Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124852024-02-03T17:28:06.652Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814753054Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124862024-02-03T17:28:06.652Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814796172Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124872024-02-03T17:28:06.652Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814838655Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124882024-02-03T17:28:06.652Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814881308Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124892024-02-03T17:28:06.652Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.814953872Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124902024-02-03T17:28:06.652Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.815091222Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124912024-02-03T17:28:06.652Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.815183928Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124922024-02-03T17:28:06.652Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.815232319Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124932024-02-03T17:28:06.652Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.816554894Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124942024-02-03T17:28:06.652Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.816628836Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124952024-02-03T17:28:06.652Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.816655262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124962024-02-03T17:28:06.652Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.816892715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124972024-02-03T17:28:06.652Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.817490129Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124982024-02-03T17:28:06.652Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.817578401Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
124992024-02-03T17:28:06.653Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.817679314Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125002024-02-03T17:28:06.653Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.817729628Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125012024-02-03T17:28:06.653Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.818426653Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125022024-02-03T17:28:06.653Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.818550303Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125032024-02-03T17:28:06.653Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.818577542Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125042024-02-03T17:28:06.653Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.818619719Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125052024-02-03T17:28:06.653Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.818644508Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125062024-02-03T17:28:06.653Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.818689665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125072024-02-03T17:28:06.653Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.818723068Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125082024-02-03T17:28:06.658Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819403103Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125092024-02-03T17:28:06.658Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819461398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125102024-02-03T17:28:06.658Z{"msg":"[0]R flush_numbers[0..12]: [207, 184, 199, 207, 204, 202, 205, 205, 207, 207, 194, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819496151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125112024-02-03T17:28:06.658Z{"msg":"[0]R generation[0..12]: [114, 103, 108, 114, 112, 110, 112, 112, 114, 114, 106, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819543876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125122024-02-03T17:28:06.658Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.8195708Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125132024-02-03T17:28:06.658Z{"msg":"[1]R flush_numbers[0..12]: [207, 184, 199, 207, 204, 202, 205, 205, 207, 207, 194, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819595378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125142024-02-03T17:28:06.658Z{"msg":"[1]R generation[0..12]: [114, 103, 108, 114, 112, 110, 112, 112, 114, 114, 106, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819636847Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125152024-02-03T17:28:06.658Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819662493Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125162024-02-03T17:28:06.658Z{"msg":"[2]R flush_numbers[0..12]: [207, 184, 199, 207, 204, 202, 205, 205, 207, 207, 194, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819685719Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125172024-02-03T17:28:06.658Z{"msg":"[2]R generation[0..12]: [114, 103, 108, 114, 112, 110, 112, 112, 114, 114, 106, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819733303Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125182024-02-03T17:28:06.658Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819758914Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125192024-02-03T17:28:06.659Z{Wait for a query_work_queue command to finish before sending IO
125202024-02-03T17:28:06.659Z"msg":"Max found gen is 115","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819784316Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125212024-02-03T17:28:06.659Z{"msg":"Generation requested: 116 >= found:115","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819824705Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125222024-02-03T17:28:06.659Z{"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819847771Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125232024-02-03T17:28:06.659Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819869313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125242024-02-03T17:28:06.659Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819910071Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125252024-02-03T17:28:06.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.81993199Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125262024-02-03T17:28:06.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819954186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125272024-02-03T17:28:06.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.819998378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125282024-02-03T17:28:06.659Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.820022501Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125292024-02-03T17:28:06.659Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.820043599Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125302024-02-03T17:28:06.659Z{"msg":"46296428-9219-442d-805f-049e9b0a2534 is now active with session: 19e3d758-42ed-451a-b2db-674cef36797b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.820083917Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125312024-02-03T17:28:06.659Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.82010554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599,"session_id":"19e3d758-42ed-451a-b2db-674cef36797b"}
125322024-02-03T17:28:06.660Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:05.820126143Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599}
125332024-02-03T17:28:06.660ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125342024-02-03T17:28:06.660ZLoading write count information from file "/tmp/test_repair_verify.data"
125352024-02-03T17:28:06.660ZRun Repair workload
125362024-02-03T17:28:06.660Z01/30 Write block 136 len 36864 data: 17 18 19 17 13 12 10 5 5
125372024-02-03T17:28:06.660Z02/30 Write block 496 len 4096 data: 7
125382024-02-03T17:28:06.660Z03/30 Write block 389 len 36864 data: 7 7 7 6 7 8 10 10 7
125392024-02-03T17:28:06.664Z04/30 Read block 38 len 16384
125402024-02-03T17:28:06.664Z05/30 Write block 565 len 28672 data: 9 11 11 8 8 8 7
125412024-02-03T17:28:06.664Z06/30 Flush
125422024-02-03T17:28:06.667Z07/30 Write block 530 len 40960 data: 11 9 9 6 5 8 9 8 7 7
125432024-02-03T17:28:06.667Z08/30 Flush
125442024-02-03T17:28:06.671Z09/30 Write block 198 len 16384 data: 6 7 7 6
125452024-02-03T17:28:06.671Z10/30 Read block 445 len 4096
125462024-02-03T17:28:06.674Z11/30 Write block 318 len 20480 data: 5 9 7 7 8
125472024-02-03T17:28:06.674Z12/30 Write block 520 len 12288 data: 5 5 6
125482024-02-03T17:28:06.674Z13/30 Flush
125492024-02-03T17:28:06.688Z14/30 Write block 506 len 32768 data: 18 19 16 17 13 10 8 6
125502024-02-03T17:28:06.694Z15/30 Write block 490 len 40960 data: 7 7 7 10 9 7 8 8 10 11
125512024-02-03T17:28:06.694Z16/30 Read block 211 len 32768
125522024-02-03T17:28:06.694Z17/30 Read block 424 len 12288
125532024-02-03T17:28:06.694Z18/30 Write block 517 len 8192 data: 7 7
125542024-02-03T17:28:06.694Z19/30 Write block 113 len 16384 data: 13 15 14 14
125552024-02-03T17:28:06.694Z20/30 Write block 193 len 28672 data: 5 5 6 8 6 7 8
125562024-02-03T17:28:06.694Z21/30 Write block 71 len 28672 data: 5 4 5 5 6 8 9
125572024-02-03T17:28:06.694Z22/30 Read block 454 len 12288
125582024-02-03T17:28:06.694Z23/30 Read block 284 len 28672
125592024-02-03T17:28:06.694Z24/30 Write block 55 len 24576 data: 8 12 10 11 12 12
125602024-02-03T17:28:06.694Z25/30 Flush
125612024-02-03T17:28:06.698Z26/30 Write block 244 len 4096 data: 5
125622024-02-03T17:28:06.698Z27/30 Write block 14 len 24576 data: 8 6 7 6 6 5
125632024-02-03T17:28:06.698Z28/30 Read block 557 len 8192
125642024-02-03T17:28:06.701Z29/30 Write block 262 len 20480 data: 11 9 10 11 13
125652024-02-03T17:28:06.701Z30/30 Read block 453 len 20480
125662024-02-03T17:28:06.708Z----------------------------------------------------------------
125672024-02-03T17:28:06.708Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
125682024-02-03T17:28:06.709ZStates: Active Active Active
125692024-02-03T17:28:06.709ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125702024-02-03T17:28:06.709Z 1 Acked 1000 Write 9 Done Done Done false
125712024-02-03T17:28:06.709Z 2 Acked 1001 Write 1 Sent Done Done false
125722024-02-03T17:28:06.709Z 3 Acked 1002 Write 9 Sent Done Done false
125732024-02-03T17:28:06.709Z 4 Acked 1003 Read 4 Sent Done Done false
125742024-02-03T17:28:06.709Z 5 Acked 1004 Write 7 Sent Done Done false
125752024-02-03T17:28:06.709Z 6 Acked 1005 Flush 0 Sent Done Done false
125762024-02-03T17:28:06.709Z 7 Acked 1006 Write 10 Sent Done Done false
125772024-02-03T17:28:06.709Z 8 Acked 1007 Flush 0 Sent Done Done false
125782024-02-03T17:28:06.709Z 9 Acked 1008 Write 4 Sent Done Done false
125792024-02-03T17:28:06.709Z 10 Acked 1009 Read 1 Sent Done Done false
125802024-02-03T17:28:06.709Z 11 Acked 1010 Write 5 Sent Done Done false
125812024-02-03T17:28:06.710Z 12 Acked 1011 Write 3 Sent Done Done false
125822024-02-03T17:28:06.710Z 13 Acked 1012 Flush 0 Sent Done Done false
125832024-02-03T17:28:06.710Z 14 Acked 1013 Write 8 Sent Done Done false
125842024-02-03T17:28:06.710Z 15 Acked 1014 Write 10 Sent Done Done false
125852024-02-03T17:28:06.710Z 16 Acked 1015 Read 8 Sent Done Done false
125862024-02-03T17:28:06.710Z 17 Acked 1016 Read 3 Sent Done Done false
125872024-02-03T17:28:06.710Z 18 Acked 1017 Write 2 Sent Done Done false
125882024-02-03T17:28:06.710Z 19 Acked 1018 Write 4 Sent Done Done false
125892024-02-03T17:28:06.710Z 20 Acked 1019 Write 7 Sent Done Done false
125902024-02-03T17:28:06.710Z 21 Acked 1020 Write 7 Sent Done Done false
125912024-02-03T17:28:06.710Z 22 Acked 1021 Read 3 Sent Done Done false
125922024-02-03T17:28:06.710Z 23 Acked 1022 Read 7 Sent Done Done false
125932024-02-03T17:28:06.710Z 24 Acked 1023 Write 6 Sent Done Done false
125942024-02-03T17:28:06.711Z 25 Acked 1024 Flush 0 Sent Done Done false
125952024-02-03T17:28:06.711Z 26 Acked 1025 Write 1 Sent Done Done false
125962024-02-03T17:28:06.711Z 27 Acked 1026 Write 6 Sent Done Done false
125972024-02-03T17:28:06.711Z 28 Acked 1027 Read 2 Sent Done Done false
125982024-02-03T17:28:06.711Z 29 Acked 1028 Write 5 Sent Done Done false
125992024-02-03T17:28:06.711Z 30 Acked 1029 Read 5 Sent Sent Done false
126002024-02-03T17:28:06.711Z STATES DS:0 DS:1 DS:2 TOTAL
126012024-02-03T17:28:06.711Z New 0 0 0 0
126022024-02-03T17:28:06.711Z Sent 29 1 0 30
126032024-02-03T17:28:06.711Z Done 1 29 30 60
126042024-02-03T17:28:06.711Z Skipped 0 0 0 0
126052024-02-03T17:28:06.711Z Error 0 0 0 0
126062024-02-03T17:28:06.712ZLast Flush: 0 1024 1024
126072024-02-03T17:28:06.712ZDownstairs last five completed:
126082024-02-03T17:28:06.712ZUpstairs last five completed: 30 29 28 27 26
126092024-02-03T17:28:06.712Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:05.871604864Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1599}
126102024-02-03T17:28:06.712ZWrote out file "/tmp/test_repair_verify.data"
126112024-02-03T17:28:06.716Z
126122024-02-03T17:28:06.719ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126132024-02-03T17:28:06.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126142024-02-03T17:28:06.745Z 0 000-019 114 114 114 207 207 207 F T T <---
126152024-02-03T17:28:06.745Z 1 020-039 103 103 103 184 184 184 F F F
126162024-02-03T17:28:06.745Z 2 040-059 108 116 116 199 211 211 F F F <---
126172024-02-03T17:28:06.745Z 3 060-079 114 116 116 207 211 211 F F F <---
126182024-02-03T17:28:06.745Z 4 080-099 112 112 112 204 204 204 F F F
126192024-02-03T17:28:06.745Z 5 100-119 110 116 116 202 211 211 F F F <---
126202024-02-03T17:28:06.749Z 6 120-139 112 116 116 205 208 208 T F F <---
126212024-02-03T17:28:06.749Z 7 140-159 112 116 116 205 208 208 T F F <---
126222024-02-03T17:28:06.749Z 8 160-179 114 114 114 207 207 207 F F F
126232024-02-03T17:28:06.749Z 9 180-199 114 116 116 207 211 211 F F F <---
126242024-02-03T17:28:06.749Z 10 200-219 106 116 116 194 210 210 F F F <---
126252024-02-03T17:28:06.749Z 11 220-239 112 112 112 205 205 205 F F F
126262024-02-03T17:28:06.749Z 12 240-259 114 114 114 207 207 207 F T T <---
126272024-02-03T17:28:06.749Z 13 260-279 114 114 114 207 207 207 F T T <---
126282024-02-03T17:28:06.749Z 14 280-299 112 112 112 205 205 205 F F F
126292024-02-03T17:28:06.749Z 15 300-319 112 116 116 205 210 210 F F F <---
126302024-02-03T17:28:06.749Z 16 320-339 109 116 116 201 210 210 F F F <---
126312024-02-03T17:28:06.750Z 17 340-359 110 110 110 202 202 202 F F F
126322024-02-03T17:28:06.750Z 18 360-379 112 112 112 203 203 203 F F F
126332024-02-03T17:28:06.750Z 19 380-399 114 116 116 207 208 208 F F F <---
126342024-02-03T17:28:06.750Z 20 400-419 113 113 113 206 206 206 F F F
126352024-02-03T17:28:06.750Z 21 420-439 114 114 114 207 207 207 F F F
126362024-02-03T17:28:06.750Z 22 440-459 110 110 110 202 202 202 F F F
126372024-02-03T17:28:06.750Z 23 460-479 104 104 104 185 185 185 F F F
126382024-02-03T17:28:06.750Z 24 480-499 112 116 116 205 211 211 F F F <---
126392024-02-03T17:28:06.750Z 25 500-519 114 116 116 207 211 211 F F F <---
126402024-02-03T17:28:06.750Z 26 520-539 114 116 116 207 210 210 F F F <---
126412024-02-03T17:28:06.750Z 27 540-559 114 114 114 207 207 207 F F F
126422024-02-03T17:28:06.750Z 28 560-579 112 116 116 204 208 208 F F F <---
126432024-02-03T17:28:06.750Z 29 580-599 114 114 114 207 207 207 F F F
126442024-02-03T17:28:06.751ZMax gen: 116, Max flush: 211
126452024-02-03T17:28:06.751ZError: Difference in extent metadata found!
126462024-02-03T17:28:06.754ZA Difference in extent metadata is expected here
126472024-02-03T17:28:06.754ZOn loop 58
126482024-02-03T17:28:06.754Z
126492024-02-03T17:28:06.758ZVerifying data now
126502024-02-03T17:28:13.176ZNo change after verify
126512024-02-03T17:28:13.176ZLoop: 58 Downstairs dump after verify (and repair):
126522024-02-03T17:28:13.200ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126532024-02-03T17:28:13.200Z 0 000-019 117 117 117 212 212 212 F F F
126542024-02-03T17:28:13.200Z 1 020-039 103 103 103 184 184 184 F F F
126552024-02-03T17:28:13.200Z 2 040-059 116 116 116 211 211 211 F F F
126562024-02-03T17:28:13.200Z 3 060-079 116 116 116 211 211 211 F F F
126572024-02-03T17:28:13.203Z 4 080-099 112 112 112 204 204 204 F F F
126582024-02-03T17:28:13.203Z 5 100-119 116 116 116 211 211 211 F F F
126592024-02-03T17:28:13.204Z 6 120-139 116 116 116 208 208 208 F F F
126602024-02-03T17:28:13.204Z 7 140-159 116 116 116 208 208 208 F F F
126612024-02-03T17:28:13.204Z 8 160-179 114 114 114 207 207 207 F F F
126622024-02-03T17:28:13.204Z 9 180-199 116 116 116 211 211 211 F F F
126632024-02-03T17:28:13.204Z 10 200-219 116 116 116 210 210 210 F F F
126642024-02-03T17:28:13.204Z 11 220-239 112 112 112 205 205 205 F F F
126652024-02-03T17:28:13.204Z 12 240-259 117 117 117 212 212 212 F F F
126662024-02-03T17:28:13.204Z 13 260-279 117 117 117 212 212 212 F F F
126672024-02-03T17:28:13.204Z 14 280-299 112 112 112 205 205 205 F F F
126682024-02-03T17:28:13.204Z 15 300-319 116 116 116 210 210 210 F F F
126692024-02-03T17:28:13.204Z 16 320-339 116 116 116 210 210 210 F F F
126702024-02-03T17:28:13.204Z 17 340-359 110 110 110 202 202 202 F F F
126712024-02-03T17:28:13.204Z 18 360-379 112 112 112 203 203 203 F F F
126722024-02-03T17:28:13.204Z 19 380-399 116 116 116 208 208 208 F F F
126732024-02-03T17:28:13.204Z 20 400-419 113 113 113 206 206 206 F F F
126742024-02-03T17:28:13.204Z 21 420-439 114 114 114 207 207 207 F F F
126752024-02-03T17:28:13.204Z 22 440-459 110 110 110 202 202 202 F F F
126762024-02-03T17:28:13.204Z 23 460-479 104 104 104 185 185 185 F F F
126772024-02-03T17:28:13.204Z 24 480-499 116 116 116 211 211 211 F F F
126782024-02-03T17:28:13.204Z 25 500-519 116 116 116 211 211 211 F F F
126792024-02-03T17:28:13.204Z 26 520-539 116 116 116 210 210 210 F F F
126802024-02-03T17:28:13.204Z 27 540-559 114 114 114 207 207 207 F F F
126812024-02-03T17:28:13.204Z 28 560-579 116 116 116 208 208 208 F F F
126822024-02-03T17:28:13.204Z 29 580-599 114 114 114 207 207 207 F F F
126832024-02-03T17:28:13.205ZMax gen: 117, Max flush: 212
126842024-02-03T17:28:13.209Z
126852024-02-03T17:28:13.209ZBegin loop 59
126862024-02-03T17:28:13.209ZDownstairs to restart: 2
126872024-02-03T17:28:13.241Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.410877308Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607}
126882024-02-03T17:28:13.241Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.411394611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607}
126892024-02-03T17:28:13.246Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.411441543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607}
126902024-02-03T17:28:13.246Z{"msg":"Crucible 9a4f3f55-fee6-4c9a-beff-394f3c1ec3b4 has session id: ff8c8a91-bbc4-4eab-bf95-e3957c5d177a","v":0,"name":"crucible","level":30Crucible runtime is spawned
126912024-02-03T17:28:13.246Z,"time":"2024-02-03T17:28:12.411524299Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
126922024-02-03T17:28:13.246Z{"msg":"Upstairs opts: Upstairs UUID: 9a4f3f55-fee6-4c9a-beff-394f3c1ec3b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.41172564Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
126932024-02-03T17:28:13.246Z{"msg":"Crucible stats registered with UUID: 9a4f3f55-fee6-4c9a-beff-394f3c1ec3b4","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.411801096Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
126942024-02-03T17:28:13.246Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.411844481Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607}
126952024-02-03T17:28:13.246Z{"msg":"9a4f3f55-fee6-4c9a-beff-394f3c1ec3b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.411896572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
126962024-02-03T17:28:13.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.411935998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
126972024-02-03T17:28:13.246Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.411999512Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
126982024-02-03T17:28:13.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412043266Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
126992024-02-03T17:28:13.246Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412084552Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127002024-02-03T17:28:13.246Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412127566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127012024-02-03T17:28:13.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412170517Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127022024-02-03T17:28:13.247Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412212943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127032024-02-03T17:28:13.247Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412270684Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127042024-02-03T17:28:13.247Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412353579Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127052024-02-03T17:28:13.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412466683Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127062024-02-03T17:28:13.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412618688Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127072024-02-03T17:28:13.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.412649439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127082024-02-03T17:28:13.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.414099637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127092024-02-03T17:28:13.247Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.41419051Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127102024-02-03T17:28:13.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.414237442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127112024-02-03T17:28:13.247Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.414323685Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127122024-02-03T17:28:13.247Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.414954194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127132024-02-03T17:28:13.252Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.415258533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127142024-02-03T17:28:13.252Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.41530011Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127152024-02-03T17:28:13.253Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.415408015Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127162024-02-03T17:28:13.253Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.415835316Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127172024-02-03T17:28:13.253Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.415915879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127182024-02-03T17:28:13.253Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.415943944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127192024-02-03T17:28:13.253Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.416331197Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127202024-02-03T17:28:13.253Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.4163998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127212024-02-03T17:28:13.253Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.416490005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127222024-02-03T17:28:13.253Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.416595409Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127232024-02-03T17:28:13.253Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417418992Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127242024-02-03T17:28:13.253Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417504011Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127252024-02-03T17:28:13.253Z{"msg":"[0]R flush_numbers[0..12]: [212, 184, 211, 211, 204, 211, 208, 208, 207, 211, 210, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417579866Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127262024-02-03T17:28:13.253Z{"msg":"[0]R generation[0..12]: [117, 103, 116, 116, 112, 116, 116, 116, 114, 116, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417641218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127272024-02-03T17:28:13.253Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.41766849Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127282024-02-03T17:28:13.253Z{"msg":"[1]R flush_numbers[0..12]: [212, 184, 211, 211, 204, 211, 208, 208, 207, 211, 210, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417718619Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127292024-02-03T17:28:13.253Z{"msg":"[1]R generation[0..12]: [117, 103, 116, 116, 112, 116, 116, 116, 114, 116, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.41774398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127302024-02-03T17:28:13.253Z{"msg":"Wait for a query_work_queue command to finish before sending IO
127312024-02-03T17:28:13.253Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417811527Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127322024-02-03T17:28:13.253Z{"msg":"[2]R flush_numbers[0..12]: [212, 184, 211, 211, 204, 211, 208, 208, 207, 211, 210, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417842176Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127332024-02-03T17:28:13.253Z{"msg":"[2]R generation[0..12]: [117, 103, 116, 116, 112, 116, 116, 116, 114, 116, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.41788698Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127342024-02-03T17:28:13.253Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417912629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127352024-02-03T17:28:13.253Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.41793497Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127362024-02-03T17:28:13.253Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.417957498Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127372024-02-03T17:28:13.253Z{"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418002471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127382024-02-03T17:28:13.254Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.4180263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127392024-02-03T17:28:13.254Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418048349Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127402024-02-03T17:28:13.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418073734Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127412024-02-03T17:28:13.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418098627Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127422024-02-03T17:28:13.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418120862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127432024-02-03T17:28:13.254Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418158936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127442024-02-03T17:28:13.254Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418194372Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127452024-02-03T17:28:13.254Z{"msg":"9a4f3f55-fee6-4c9a-beff-394f3c1ec3b4 is now active with session: ff8c8a91-bbc4-4eab-bf95-e3957c5d177a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418241455Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127462024-02-03T17:28:13.254Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418272976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607,"session_id":"ff8c8a91-bbc4-4eab-bf95-e3957c5d177a"}
127472024-02-03T17:28:13.254Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:12.418332637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607}
127482024-02-03T17:28:13.254ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127492024-02-03T17:28:13.254ZLoading write count information from file "/tmp/test_repair_verify.data"
127502024-02-03T17:28:13.254ZRun Repair workload
127512024-02-03T17:28:13.254Z01/30 Flush
127522024-02-03T17:28:13.258Z02/30 Write block 441 len 24576 data: 13 8 8 7 9 8
127532024-02-03T17:28:13.258Z03/30 Read block 453 len 40960
127542024-02-03T17:28:13.258Z04/30 Flush
127552024-02-03T17:28:13.261Z05/30 Write block 413 len 20480 data: 10 9 5 6 8
127562024-02-03T17:28:13.261Z06/30 Read block 253 len 16384
127572024-02-03T17:28:13.264Z07/30 Read block 533 len 20480
127582024-02-03T17:28:13.264Z08/30 Flush
127592024-02-03T17:28:13.267Z09/30 Write block 578 len 8192 data: 13 13
127602024-02-03T17:28:13.270Z10/30 Write block 428 len 36864 data: 6 4 7 7 7 5 7 8 8
127612024-02-03T17:28:13.270Z11/30 Read block 406 len 4096
127622024-02-03T17:28:13.274Z12/30 Write block 567 len 28672 data: 12 9 9 9 8 5 5
127632024-02-03T17:28:13.274Z13/30 Flush
127642024-02-03T17:28:13.278Z14/30 Flush
127652024-02-03T17:28:13.281Z15/30 Write block 145 len 12288 data: 6 5 4
127662024-02-03T17:28:13.281Z16/30 Read block 82 len 32768
127672024-02-03T17:28:13.281Z17/30 Write block 430 len 8192 data: 8 8
127682024-02-03T17:28:13.284Z18/30 Read block 580 len 36864
127692024-02-03T17:28:13.284Z19/30 Read block 274 len 28672
127702024-02-03T17:28:13.290Z20/30 Read block 23 len 40960
127712024-02-03T17:28:13.306Z21/30 Write block 89 len 20480 data: 9 10 9 10 10
127722024-02-03T17:28:13.306Z22/30 Read block 91 len 40960
127732024-02-03T17:28:13.306Z23/30 Read block 90 len 24576
127742024-02-03T17:28:13.306Z24/30 Write block 356 len 36864 data: 16 15 15 16 15 16 14 16 14
127752024-02-03T17:28:13.306Z25/30 Write block 451 len 8192 data: 6 7
127762024-02-03T17:28:13.306Z26/30 Read block 86 len 16384
127772024-02-03T17:28:13.311Z27/30 Read block 241 len 36864
127782024-02-03T17:28:13.311Z28/30 Write block 116 len 24576 data: 15 15 13 13 12 12
127792024-02-03T17:28:13.311Z29/30 Write block 335 len 12288 data: 8 8 7
127802024-02-03T17:28:13.311Z30/30 Write block 512 len 8192 data: 9 7
127812024-02-03T17:28:13.311Z----------------------------------------------------------------
127822024-02-03T17:28:13.311Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:29
127832024-02-03T17:28:13.311ZStates: Active Active Active
127842024-02-03T17:28:13.311ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127852024-02-03T17:28:13.311Z 2 Acked 1001 Write 6 Done Done Done false
127862024-02-03T17:28:13.311Z 3 Acked 1002 Read 10 Done Done Done false
127872024-02-03T17:28:13.311Z 4 Acked 1003 Flush 0 Done Done Sent false
127882024-02-03T17:28:13.311Z 5 Acked 1004 Write 5 Done Done Sent false
127892024-02-03T17:28:13.311Z 6 Acked 1005 Read 4 Done Done Sent false
127902024-02-03T17:28:13.311Z 7 Acked 1006 Read 5 Done Done Sent false
127912024-02-03T17:28:13.311Z 8 Acked 1007 Flush 0 Done Done Sent false
127922024-02-03T17:28:13.312Z 9 Acked 1008 Write 2 Done Done Sent false
127932024-02-03T17:28:13.312Z 10 Acked 1009 Write 9 Done Done Sent false
127942024-02-03T17:28:13.312Z 11 Acked 1010 Read 1 Done Done Sent false
127952024-02-03T17:28:13.312Z 12 Acked 1011 Write 7 Done Done Sent false
127962024-02-03T17:28:13.312Z 13 Acked 1012 Flush 0 Done Done Sent false
127972024-02-03T17:28:13.312Z 14 Acked 1013 Flush 0 Done Done Sent false
127982024-02-03T17:28:13.312Z 15 Acked 1014 Write 3 Done Done Sent false
127992024-02-03T17:28:13.312Z 16 Acked 1015 Read 8 Done Done Sent false
128002024-02-03T17:28:13.312Z 17 Acked 1016 Write 2 Done Done Sent false
128012024-02-03T17:28:13.312Z 18 Acked 1017 Read 9 Done Done Sent false
128022024-02-03T17:28:13.312Z 19 Acked 1018 Read 7 Done Done Sent false
128032024-02-03T17:28:13.312Z 20 Acked 1019 Read 10 Done Done Sent false
128042024-02-03T17:28:13.312Z 21 Acked 1020 Write 5 Done Done Sent false
128052024-02-03T17:28:13.312Z 22 Acked 1021 Read 10 Done Done Sent false
128062024-02-03T17:28:13.312Z 23 Acked 1022 Read 6 Done Done Sent false
128072024-02-03T17:28:13.312Z 24 Acked 1023 Write 9 Done Done Sent false
128082024-02-03T17:28:13.312Z 25 Acked 1024 Write 2 Done Done Sent false
128092024-02-03T17:28:13.312Z 26 Acked 1025 Read 4 Done Done Sent false
128102024-02-03T17:28:13.312Z 27 Acked 1026 Read 9 Done Done Sent false
128112024-02-03T17:28:13.312Z 28 Acked 1027 Write 6 Done Done Sent false
128122024-02-03T17:28:13.312Z 29 Acked 1028 Write 3 Done Sent Sent false
128132024-02-03T17:28:13.312Z 30 Acked 1029 Write 2 Sent Sent Sent false
128142024-02-03T17:28:13.312Z STATES DS:0 DS:1 DS:2 TOTAL
128152024-02-03T17:28:13.312Z New 0 0 0 0
128162024-02-03T17:28:13.312Z Sent 1 2 27 30
128172024-02-03T17:28:13.313Z Done 28 27 2 57
128182024-02-03T17:28:13.313Z Skipped 0 0 0 0
128192024-02-03T17:28:13.313Z Error 0 0 0 0
128202024-02-03T17:28:13.313ZLast Flush: 1013 1013 1000
128212024-02-03T17:28:13.313ZDownstairs last five completed: 1000
128222024-02-03T17:28:13.313ZUpstairs last five completed: 30 29 28 27 26
128232024-02-03T17:28:13.313Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:12.470797479Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1607}
128242024-02-03T17:28:13.313ZWrote out file "/tmp/test_repair_verify.data"
128252024-02-03T17:28:13.313Z
128262024-02-03T17:28:13.317ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128272024-02-03T17:28:13.346ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128282024-02-03T17:28:13.346Z 0 000-019 117 117 117 212 212 212 F F F
128292024-02-03T17:28:13.346Z 1 020-039 103 103 103 184 184 184 F F F
128302024-02-03T17:28:13.346Z 2 040-059 116 116 116 211 211 211 F F F
128312024-02-03T17:28:13.346Z 3 060-079 116 116 116 211 211 211 F F F
128322024-02-03T17:28:13.346Z 4 080-099 112 112 112 204 204 204 T T F <---
128332024-02-03T17:28:13.346Z 5 100-119 116 116 116 211 211 211 T T F <---
128342024-02-03T17:28:13.347Z 6 120-139 116 116 116 208 208 208 T T F <---
128352024-02-03T17:28:13.350Z 7 140-159 116 116 116 208 208 208 T T F <---
128362024-02-03T17:28:13.350Z 8 160-179 114 114 114 207 207 207 F F F
128372024-02-03T17:28:13.350Z 9 180-199 116 116 116 211 211 211 F F F
128382024-02-03T17:28:13.350Z 10 200-219 116 116 116 210 210 210 F F F
128392024-02-03T17:28:13.350Z 11 220-239 112 112 112 205 205 205 F F F
128402024-02-03T17:28:13.350Z 12 240-259 117 117 117 212 212 212 F F F
128412024-02-03T17:28:13.350Z 13 260-279 117 117 117 212 212 212 F F F
128422024-02-03T17:28:13.350Z 14 280-299 112 112 112 205 205 205 F F F
128432024-02-03T17:28:13.350Z 15 300-319 116 116 116 210 210 210 F F F
128442024-02-03T17:28:13.350Z 16 320-339 116 116 116 210 210 210 T T F <---
128452024-02-03T17:28:13.350Z 17 340-359 110 110 110 202 202 202 T T F <---
128462024-02-03T17:28:13.351Z 18 360-379 112 112 112 203 203 203 T T F <---
128472024-02-03T17:28:13.351Z 19 380-399 116 116 116 208 208 208 F F F
128482024-02-03T17:28:13.351Z 20 400-419 118 118 113 215 215 206 F F F <---
128492024-02-03T17:28:13.351Z 21 420-439 118 118 114 216 216 207 T T F <---
128502024-02-03T17:28:13.351Z 22 440-459 118 118 110 214 214 202 T T T <---
128512024-02-03T17:28:13.351Z 23 460-479 104 104 104 185 185 185 F F F
128522024-02-03T17:28:13.351Z 24 480-499 116 116 116 211 211 211 F F F
128532024-02-03T17:28:13.351Z 25 500-519 116 116 116 211 211 211 T T F <---
128542024-02-03T17:28:13.351Z 26 520-539 116 116 116 210 210 210 F F F
128552024-02-03T17:28:13.351Z 27 540-559 114 114 114 207 207 207 F F F
128562024-02-03T17:28:13.351Z 28 560-579 118 118 116 216 216 208 F F F <---
128572024-02-03T17:28:13.351Z 29 580-599 114 114 114 207 207 207 F F F
128582024-02-03T17:28:13.351ZMax gen: 118, Max flush: 216
128592024-02-03T17:28:13.351ZError: Difference in extent metadata found!
128602024-02-03T17:28:13.355ZA Difference in extent metadata is expected here
128612024-02-03T17:28:13.355ZOn loop 59
128622024-02-03T17:28:13.355Z
128632024-02-03T17:28:13.359ZVerifying data now
128642024-02-03T17:28:18.443ZNo change after verify
128652024-02-03T17:28:18.443ZLoop: 59 Downstairs dump after verify (and repair):
128662024-02-03T17:28:18.468ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128672024-02-03T17:28:18.468Z 0 000-019 117 117 117 212 212 212 F F F
128682024-02-03T17:28:18.468Z 1 020-039 103 103 103 184 184 184 F F F
128692024-02-03T17:28:18.468Z 2 040-059 116 116 116 211 211 211 F F F
128702024-02-03T17:28:18.468Z 3 060-079 116 116 116 211 211 211 F F F
128712024-02-03T17:28:18.472Z 4 080-099 119 119 119 217 217 217 F F F
128722024-02-03T17:28:18.472Z 5 100-119 119 119 119 217 217 217 F F F
128732024-02-03T17:28:18.472Z 6 120-139 119 119 119 217 217 217 F F F
128742024-02-03T17:28:18.472Z 7 140-159 119 119 119 217 217 217 F F F
128752024-02-03T17:28:18.472Z 8 160-179 114 114 114 207 207 207 F F F
128762024-02-03T17:28:18.473Z 9 180-199 116 116 116 211 211 211 F F F
128772024-02-03T17:28:18.473Z 10 200-219 116 116 116 210 210 210 F F F
128782024-02-03T17:28:18.473Z 11 220-239 112 112 112 205 205 205 F F F
128792024-02-03T17:28:18.473Z 12 240-259 117 117 117 212 212 212 F F F
128802024-02-03T17:28:18.473Z 13 260-279 117 117 117 212 212 212 F F F
128812024-02-03T17:28:18.473Z 14 280-299 112 112 112 205 205 205 F F F
128822024-02-03T17:28:18.473Z 15 300-319 116 116 116 210 210 210 F F F
128832024-02-03T17:28:18.473Z 16 320-339 119 119 119 217 217 217 F F F
128842024-02-03T17:28:18.473Z 17 340-359 119 119 119 217 217 217 F F F
128852024-02-03T17:28:18.473Z 18 360-379 119 119 119 217 217 217 F F F
128862024-02-03T17:28:18.473Z 19 380-399 116 116 116 208 208 208 F F F
128872024-02-03T17:28:18.473Z 20 400-419 118 118 118 215 215 215 F F F
128882024-02-03T17:28:18.473Z 21 420-439 119 119 119 217 217 217 F F F
128892024-02-03T17:28:18.474Z 22 440-459 119 119 119 217 217 217 F F F
128902024-02-03T17:28:18.474Z 23 460-479 104 104 104 185 185 185 F F F
128912024-02-03T17:28:18.474Z 24 480-499 116 116 116 211 211 211 F F F
128922024-02-03T17:28:18.474Z 25 500-519 119 119 119 217 217 217 F F F
128932024-02-03T17:28:18.474Z 26 520-539 116 116 116 210 210 210 F F F
128942024-02-03T17:28:18.474Z 27 540-559 114 114 114 207 207 207 F F F
128952024-02-03T17:28:18.474Z 28 560-579 118 118 118 216 216 216 F F F
128962024-02-03T17:28:18.474Z 29 580-599 114 114 114 207 207 207 F F F
128972024-02-03T17:28:18.474ZMax gen: 119, Max flush: 217
128982024-02-03T17:28:18.501Z
128992024-02-03T17:28:18.501ZBegin loop 60
129002024-02-03T17:28:18.501ZDownstairs to restart: 2
129012024-02-03T17:28:18.508Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.677949589Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615}
129022024-02-03T17:28:18.508Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.678418043Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615}
129032024-02-03T17:28:18.509Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.678449234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615}
129042024-02-03T17:28:18.525Z{"msg":"Crucible ee27c450-104d-4057-876c-b40affc83ad2 has session id: 57adb6c7-f3c9-44f3-bcb6-112f43d079f3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.678486932Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129052024-02-03T17:28:18.525Z{"msg":"Upstairs opts: Upstairs UUID: ee27c450-104d-4057-876c-b40affc83ad2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.678576375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129062024-02-03T17:28:18.525Z{"msg":"Crucible stats registered with UUID: ee27c450-104d-4057-876c-b40affc83ad2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.67861974Z","hostname":"Crucible runtime is spawned
129072024-02-03T17:28:18.525Zip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129082024-02-03T17:28:18.525Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.678841222Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615}
129092024-02-03T17:28:18.525Z{"msg":"ee27c450-104d-4057-876c-b40affc83ad2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679014455Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129102024-02-03T17:28:18.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679068956Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129112024-02-03T17:28:18.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679095337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129122024-02-03T17:28:18.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679118445Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129132024-02-03T17:28:18.525Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.67918635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129142024-02-03T17:28:18.525Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679228442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129152024-02-03T17:28:18.526Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.67927286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129162024-02-03T17:28:18.526Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679315348Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129172024-02-03T17:28:18.526Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.67936074Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129182024-02-03T17:28:18.526Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.67940167Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129192024-02-03T17:28:18.526Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:17.679579037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129202024-02-03T17:28:18.526Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:17.679694675Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129212024-02-03T17:28:18.526Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:17.679773651Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129222024-02-03T17:28:18.526Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679831204Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129232024-02-03T17:28:18.526Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679872211Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129242024-02-03T17:28:18.526Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.67989701Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129252024-02-03T17:28:18.526Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679944576Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129262024-02-03T17:28:18.526Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.679969668Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129272024-02-03T17:28:18.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.681076137Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129282024-02-03T17:28:18.527Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.681136209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129292024-02-03T17:28:18.527Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.681170814Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129302024-02-03T17:28:18.527Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.68121202Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129312024-02-03T17:28:18.527Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.682250069Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129322024-02-03T17:28:18.527Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.682334764Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129332024-02-03T17:28:18.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.683343482Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129342024-02-03T17:28:18.532Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.68340244Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129352024-02-03T17:28:18.532Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.683428578Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129362024-02-03T17:28:18.532Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.683468547Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129372024-02-03T17:28:18.532Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.683492828Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129382024-02-03T17:28:18.532Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:17.683533325Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129392024-02-03T17:28:19.515Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.680162255Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129402024-02-03T17:28:19.515Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.680292405Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129412024-02-03T17:28:19.520Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.681489839Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129422024-02-03T17:28:19.521Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.681557593Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129432024-02-03T17:28:19.521Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.68226231Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129442024-02-03T17:28:19.521Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.68317164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129452024-02-03T17:28:19.521Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683263021Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129462024-02-03T17:28:19.521Z{"msg":"[0]R flush_numbers[0..12]: [212, 184, 211, 211, 217, 217, 217, 217, 207, 211, 210, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683314765Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129472024-02-03T17:28:19.521Z{"msg":"[0]R generation[0..12]: [117, 103, 116, 116, 119, 119, 119, 119, 114, 116, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683361244Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129482024-02-03T17:28:19.521Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683425296Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129492024-02-03T17:28:19.521Z{"msg":"[1]R flush_numbers[0..12]: [212, 184, 211, 211, 217, 217, 217, 217, 207, 211, 210, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683471702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129502024-02-03T17:28:19.521Z{"msg":"[1]R generation[0..12]: [117, 103, 116, 116, 119, 119, 119, 119, 114, 116, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.68353226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129512024-02-03T17:28:19.521Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683624327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129522024-02-03T17:28:19.521Z{"msg":"Wait for a query_work_queue command to finish before sending IO
129532024-02-03T17:28:19.521Z[2]R flush_numbers[0..12]: [212, 184, 211, 211, 217, 217, 217, 217, 207, 211, 210, 205]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683668825Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129542024-02-03T17:28:19.522Z{"msg":"[2]R generation[0..12]: [117, 103, 116, 116, 119, 119, 119, 119, 114, 116, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683778343Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129552024-02-03T17:28:19.522Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683807365Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129562024-02-03T17:28:19.522Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683829163Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129572024-02-03T17:28:19.522Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683885643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129582024-02-03T17:28:19.522Z{"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683909177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129592024-02-03T17:28:19.522Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.68393062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129602024-02-03T17:28:19.522Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.683965973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129612024-02-03T17:28:19.522Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.684029607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129622024-02-03T17:28:19.522Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.684104698Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129632024-02-03T17:28:19.522Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.684148126Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129642024-02-03T17:28:19.522Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.684195706Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129652024-02-03T17:28:19.522Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.684219174Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129662024-02-03T17:28:19.523Z{"msg":"ee27c450-104d-4057-876c-b40affc83ad2 is now active with session: 57adb6c7-f3c9-44f3-bcb6-112f43d079f3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.684251268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129672024-02-03T17:28:19.523Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.684313572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615,"session_id":"57adb6c7-f3c9-44f3-bcb6-112f43d079f3"}
129682024-02-03T17:28:19.523Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:18.684351656Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615}
129692024-02-03T17:28:19.523ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129702024-02-03T17:28:19.523ZLoading write count information from file "/tmp/test_repair_verify.data"
129712024-02-03T17:28:19.523ZRun Repair workload
129722024-02-03T17:28:19.523Z01/30 Write block 539 len 12288 data: 8 7 7
129732024-02-03T17:28:19.523Z02/30 Read block 141 len 8192
129742024-02-03T17:28:19.527Z03/30 Flush
129752024-02-03T17:28:19.527Z04/30 Write block 371 len 4096 data: 6
129762024-02-03T17:28:19.527Z05/30 Flush
129772024-02-03T17:28:19.528Z06/30 Write block 281 len 32768 data: 13 14 12 11 9 8 9 8
129782024-02-03T17:28:19.528Z07/30 Read block 474 len 20480
129792024-02-03T17:28:19.531Z08/30 Read block 66 len 4096
129802024-02-03T17:28:19.531Z09/30 Read block 53 len 24576
129812024-02-03T17:28:19.532Z10/30 Write block 581 len 24576 data: 12 11 10 10 8 10
129822024-02-03T17:28:19.532Z11/30 Write block 263 len 24576 data: 10 11 12 14 11 13
129832024-02-03T17:28:19.535Z12/30 Write block 305 len 8192 data: 11 12
129842024-02-03T17:28:19.535Z13/30 Write block 219 len 20480 data: 9 10 15 15 13
129852024-02-03T17:28:19.535Z14/30 Write block 348 len 4096 data: 9
129862024-02-03T17:28:19.536Z15/30 Flush
129872024-02-03T17:28:19.540Z16/30 Write block 156 len 36864 data: 11 12 11 12 13 14 14 12 10
129882024-02-03T17:28:19.544Z17/30 Read block 19 len 8192
129892024-02-03T17:28:19.544Z18/30 Read block 467 len 16384
129902024-02-03T17:28:19.547Z19/30 Write block 82 len 4096 data: 8
129912024-02-03T17:28:19.547Z20/30 Write block 247 len 40960 data: 8 9 9 14 9 6 5 8 8 7
129922024-02-03T17:28:19.547Z21/30 Read block 45 len 40960
129932024-02-03T17:28:19.552Z22/30 Write block 492 len 24576 data: 8 11 10 8 9 9
129942024-02-03T17:28:19.555Z23/30 Write block 431 len 28672 data: 9 8 6 8 9 9 9
129952024-02-03T17:28:19.556Z24/30 Write block 202 len 12288 data: 6 8 7
129962024-02-03T17:28:19.556Z25/30 Write block 96 len 16384 data: 8 9 6 6
129972024-02-03T17:28:19.556Z26/30 Write block 28 len 8192 data: 10 8
129982024-02-03T17:28:19.556Z27/30 Read block 31 len 12288
129992024-02-03T17:28:19.561Z28/30 Read block 0 len 36864
130002024-02-03T17:28:19.561Z29/30 Read block 492 len 28672
130012024-02-03T17:28:19.581Z30/30 Write block 121 len 16384 data: 13 9 9 6
130022024-02-03T17:28:19.581Z----------------------------------------------------------------
130032024-02-03T17:28:19.581Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:25
130042024-02-03T17:28:19.581ZStates: Active Active Active
130052024-02-03T17:28:19.581ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
130062024-02-03T17:28:19.581Z 6 Acked 1005 Write 8 Done Done Done false
130072024-02-03T17:28:19.581Z 7 Acked 1006 Read 5 Done Done Done false
130082024-02-03T17:28:19.582Z 8 Acked 1007 Read 1 Done Done Sent false
130092024-02-03T17:28:19.582Z 9 Acked 1008 Read 6 Done Done Sent false
130102024-02-03T17:28:19.582Z 10 Acked 1009 Write 6 Done Done Sent false
130112024-02-03T17:28:19.582Z 11 Acked 1010 Write 6 Done Done Sent false
130122024-02-03T17:28:19.582Z 12 Acked 1011 Write 2 Done Done Sent false
130132024-02-03T17:28:19.582Z 13 Acked 1012 Write 5 Done Done Sent false
130142024-02-03T17:28:19.582Z 14 Acked 1013 Write 1 Done Done Sent false
130152024-02-03T17:28:19.582Z 15 Acked 1014 Flush 0 Done Done Sent false
130162024-02-03T17:28:19.582Z 16 Acked 1015 Write 9 Done Done Sent false
130172024-02-03T17:28:19.582Z 17 Acked 1016 Read 2 Done Done Sent false
130182024-02-03T17:28:19.582Z 18 Acked 1017 Read 4 Done Done Sent false
130192024-02-03T17:28:19.582Z 19 Acked 1018 Write 1 Done Done Sent false
130202024-02-03T17:28:19.582Z 20 Acked 1019 Write 10 Done Done Sent false
130212024-02-03T17:28:19.583Z 21 Acked 1020 Read 10 Done Done Sent false
130222024-02-03T17:28:19.583Z 22 Acked 1021 Write 6 Done Done Sent false
130232024-02-03T17:28:19.583Z 23 Acked 1022 Write 7 Done Done Sent false
130242024-02-03T17:28:19.583Z 24 Acked 1023 Write 3 Done Done Sent false
130252024-02-03T17:28:19.583Z 25 Acked 1024 Write 4 Done Done Sent false
130262024-02-03T17:28:19.583Z 26 Acked 1025 Write 2 Done Done Sent false
130272024-02-03T17:28:19.583Z 27 Acked 1026 Read 3 Done Done Sent false
130282024-02-03T17:28:19.583Z 28 Acked 1027 Read 9 Done Done Sent false
130292024-02-03T17:28:19.583Z 29 Acked 1028 Read 7 Done Done Sent false
130302024-02-03T17:28:19.583Z 30 Acked 1029 Write 4 Sent Sent Sent false
130312024-02-03T17:28:19.583Z STATES DS:0 DS:1 DS:2 TOTAL
130322024-02-03T17:28:19.583Z New 0 0 0 0
130332024-02-03T17:28:19.583Z Sent 1 1 23 25
130342024-02-03T17:28:19.584Z Done 24 24 2 50
130352024-02-03T17:28:19.584Z Skipped 0 0 0 0
130362024-02-03T17:28:19.584Z Error 0 0 0 0
130372024-02-03T17:28:19.584ZLast Flush: 1014 1014 1004
130382024-02-03T17:28:19.584ZDownstairs last five completed: 1004 1003 1002 1001 1000
130392024-02-03T17:28:19.584ZUpstairs last five completed: 30 29 28 27 26
130402024-02-03T17:28:19.584Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:18.731230862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1615}
130412024-02-03T17:28:19.584ZWrote out file "/tmp/test_repair_verify.data"
130422024-02-03T17:28:19.588Z
130432024-02-03T17:28:19.588ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130442024-02-03T17:28:19.606ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130452024-02-03T17:28:19.606Z 0 000-019 117 117 117 212 212 212 F F F
130462024-02-03T17:28:19.606Z 1 020-039 103 103 103 184 184 184 T T F <---
130472024-02-03T17:28:19.606Z 2 040-059 116 116 116 211 211 211 F F F
130482024-02-03T17:28:19.610Z 3 060-079 116 116 116 211 211 211 F F F
130492024-02-03T17:28:19.610Z 4 080-099 119 119 119 217 217 217 T T F <---
130502024-02-03T17:28:19.610Z 5 100-119 119 119 119 217 217 217 F F F
130512024-02-03T17:28:19.610Z 6 120-139 119 119 119 217 217 217 T T F <---
130522024-02-03T17:28:19.610Z 7 140-159 119 119 119 217 217 217 T T F <---
130532024-02-03T17:28:19.610Z 8 160-179 114 114 114 207 207 207 T T F <---
130542024-02-03T17:28:19.610Z 9 180-199 116 116 116 211 211 211 F F F
130552024-02-03T17:28:19.611Z 10 200-219 120 120 116 220 220 210 T T F <---
130562024-02-03T17:28:19.611Z 11 220-239 120 120 112 220 220 205 F F F <---
130572024-02-03T17:28:19.611Z 12 240-259 117 117 117 212 212 212 T T F <---
130582024-02-03T17:28:19.611Z 13 260-279 120 120 117 220 220 212 F F F <---
130592024-02-03T17:28:19.611Z 14 280-299 120 120 112 220 220 205 F F T <---
130602024-02-03T17:28:19.611Z 15 300-319 120 120 116 220 220 210 F F F <---
130612024-02-03T17:28:19.611Z 16 320-339 119 119 119 217 217 217 F F F
130622024-02-03T17:28:19.611Z 17 340-359 120 120 119 220 220 217 F F F <---
130632024-02-03T17:28:19.611Z 18 360-379 120 120 120 219 219 219 F F F
130642024-02-03T17:28:19.611Z 19 380-399 116 116 116 208 208 208 F F F
130652024-02-03T17:28:19.611Z 20 400-419 118 118 118 215 215 215 F F F
130662024-02-03T17:28:19.611Z 21 420-439 119 119 119 217 217 217 T T F <---
130672024-02-03T17:28:19.611Z 22 440-459 119 119 119 217 217 217 F F F
130682024-02-03T17:28:19.611Z 23 460-479 104 104 104 185 185 185 F F F
130692024-02-03T17:28:19.611Z 24 480-499 116 116 116 211 211 211 T T F <---
130702024-02-03T17:28:19.611Z 25 500-519 119 119 119 217 217 217 F F F
130712024-02-03T17:28:19.611Z 26 520-539 120 120 120 218 218 218 F F F
130722024-02-03T17:28:19.611Z 27 540-559 120 120 120 218 218 218 F F F
130732024-02-03T17:28:19.611Z 28 560-579 118 118 118 216 216 216 F F F
130742024-02-03T17:28:19.611Z 29 580-599 120 120 114 220 220 207 F F F <---
130752024-02-03T17:28:19.611ZMax gen: 120, Max flush: 220
130762024-02-03T17:28:19.611ZError: Difference in extent metadata found!
130772024-02-03T17:28:19.615ZA Difference in extent metadata is expected here
130782024-02-03T17:28:19.615ZOn loop 60
130792024-02-03T17:28:19.615Z
130802024-02-03T17:28:19.619ZVerifying data now
130812024-02-03T17:28:24.920ZNo change after verify
130822024-02-03T17:28:24.921ZLoop: 60 Downstairs dump after verify (and repair):
130832024-02-03T17:28:24.951ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130842024-02-03T17:28:24.951Z 0 000-019 117 117 117 212 212 212 F F F
130852024-02-03T17:28:24.951Z 1 020-039 121 121 121 221 221 221 F F F
130862024-02-03T17:28:24.951Z 2 040-059 116 116 116 211 211 211 F F F
130872024-02-03T17:28:24.951Z 3 060-079 116 116 116 211 211 211 F F F
130882024-02-03T17:28:24.951Z 4 080-099 121 121 121 221 221 221 F F F
130892024-02-03T17:28:24.951Z 5 100-119 119 119 119 217 217 217 F F F
130902024-02-03T17:28:24.951Z 6 120-139 121 121 121 221 221 221 F F F
130912024-02-03T17:28:24.951Z 7 140-159 121 121 121 221 221 221 F F F
130922024-02-03T17:28:24.951Z 8 160-179 121 121 121 221 221 221 F F F
130932024-02-03T17:28:24.951Z 9 180-199 116 116 116 211 211 211 F F F
130942024-02-03T17:28:24.951Z 10 200-219 121 121 121 221 221 221 F F F
130952024-02-03T17:28:24.952Z 11 220-239 120 120 120 220 220 220 F F F
130962024-02-03T17:28:24.952Z 12 240-259 121 121 121 221 221 221 F F F
130972024-02-03T17:28:24.952Z 13 260-279 120 120 120 220 220 220 F F F
130982024-02-03T17:28:24.952Z 14 280-299 120 120 120 220 220 220 F F F
130992024-02-03T17:28:24.952Z 15 300-319 120 120 120 220 220 220 F F F
131002024-02-03T17:28:24.952Z 16 320-339 119 119 119 217 217 217 F F F
131012024-02-03T17:28:24.952Z 17 340-359 120 120 120 220 220 220 F F F
131022024-02-03T17:28:24.952Z 18 360-379 120 120 120 219 219 219 F F F
131032024-02-03T17:28:24.952Z 19 380-399 116 116 116 208 208 208 F F F
131042024-02-03T17:28:24.952Z 20 400-419 118 118 118 215 215 215 F F F
131052024-02-03T17:28:24.952Z 21 420-439 121 121 121 221 221 221 F F F
131062024-02-03T17:28:24.952Z 22 440-459 119 119 119 217 217 217 F F F
131072024-02-03T17:28:24.952Z 23 460-479 104 104 104 185 185 185 F F F
131082024-02-03T17:28:24.952Z 24 480-499 121 121 121 221 221 221 F F F
131092024-02-03T17:28:24.952Z 25 500-519 119 119 119 217 217 217 F F F
131102024-02-03T17:28:24.952Z 26 520-539 120 120 120 218 218 218 F F F
131112024-02-03T17:28:24.952Z 27 540-559 120 120 120 218 218 218 F F F
131122024-02-03T17:28:24.952Z 28 560-579 118 118 118 216 216 216 F F F
131132024-02-03T17:28:24.952Z 29 580-599 120 120 120 220 220 220 F F F
131142024-02-03T17:28:24.952ZMax gen: 121, Max flush: 221
131152024-02-03T17:28:24.956Z
131162024-02-03T17:28:24.956ZBegin loop 61
131172024-02-03T17:28:24.956ZDownstairs to restart: 2
131182024-02-03T17:28:24.982Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.148796486Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623}
131192024-02-03T17:28:24.993Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
131202024-02-03T17:28:24.993Z,"time":"2024-02-03T17:28:24.14930889Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623}
131212024-02-03T17:28:24.993Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.14953554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623}
131222024-02-03T17:28:24.993Z{"msg":"Crucible 5bf63fc5-e8ef-4df1-9ccf-f14fbc6d4355 has session id: 958ce1d2-af4f-44b6-8c3b-28e3664c24c8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.149566563Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131232024-02-03T17:28:24.993Z{"msg":"Upstairs opts: Upstairs UUID: 5bf63fc5-e8ef-4df1-9ccf-f14fbc6d4355, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.14963525Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131242024-02-03T17:28:24.993Z{"msg":"Crucible stats registered with UUID: 5bf63fc5-e8ef-4df1-9ccf-f14fbc6d4355","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.149668899Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131252024-02-03T17:28:24.993Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.149718016Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623}
131262024-02-03T17:28:24.993Z{"msg":"5bf63fc5-e8ef-4df1-9ccf-f14fbc6d4355 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.149739451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131272024-02-03T17:28:24.993Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.149804824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131282024-02-03T17:28:24.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.149880686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131292024-02-03T17:28:24.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.149927766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131302024-02-03T17:28:24.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150001225Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131312024-02-03T17:28:24.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150104016Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131322024-02-03T17:28:24.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150172664Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131332024-02-03T17:28:24.994Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150256189Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131342024-02-03T17:28:24.994Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150333809Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131352024-02-03T17:28:24.994Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150380192Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131362024-02-03T17:28:24.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150460198Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131372024-02-03T17:28:24.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150506578Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131382024-02-03T17:28:24.994Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.150572227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131392024-02-03T17:28:24.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.151963667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131402024-02-03T17:28:24.994Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.152032338Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131412024-02-03T17:28:24.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.152069349Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131422024-02-03T17:28:24.994Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.152109998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131432024-02-03T17:28:24.994Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.15305399Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131442024-02-03T17:28:24.994Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.153120842Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131452024-02-03T17:28:24.994Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.153463598Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131462024-02-03T17:28:24.994Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.153575434Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131472024-02-03T17:28:24.994Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.154191527Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131482024-02-03T17:28:24.994Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.154255997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131492024-02-03T17:28:24.994Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.154282489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131502024-02-03T17:28:24.994Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.154303514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131512024-02-03T17:28:24.994Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.154325105Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131522024-02-03T17:28:24.995Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.154398208Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131532024-02-03T17:28:24.995Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.154769133Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131542024-02-03T17:28:24.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.155509238Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131552024-02-03T17:28:24.995Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.155597701Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131562024-02-03T17:28:24.995Z{"msg":"[0]R flush_numbers[0..12]: [212, 221, 211, 211, 221, 217, 221, 221, 221, 211, 221, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.155734242Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131572024-02-03T17:28:24.995Z{"msg":"[0]R generation[0..12]: [117, 121, 116, 116, 121, 119, 121, 121, 121, 116, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.155813688Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131582024-02-03T17:28:24.995Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.155889456Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131592024-02-03T17:28:24.995Z{"msg":"[1]R flush_numbers[0..12]: [212, 221, 211, 211, 221, 217, 221, 221, 221, 211, 221, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.155960622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131602024-02-03T17:28:24.995Z{"msg":"[1]R generation[0..12]: [117, 121, 116, 116, 121, 119, 121, 121, 121, 116, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156035788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131612024-02-03T17:28:24.995Z{"msg":"Wait for a query_work_queue command to finish before sending IO
131622024-02-03T17:28:24.995Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156084811Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131632024-02-03T17:28:24.995Z{"msg":"[2]R flush_numbers[0..12]: [212, 221, 211, 211, 221, 217, 221, 221, 221, 211, 221, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156161608Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131642024-02-03T17:28:24.995Z{"msg":"[2]R generation[0..12]: [117, 121, 116, 116, 121, 119, 121, 121, 121, 116, 121, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156235829Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131652024-02-03T17:28:24.995Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156310705Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131662024-02-03T17:28:24.995Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.15638285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131672024-02-03T17:28:24.995Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156428465Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131682024-02-03T17:28:24.995Z{"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.15650297Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131692024-02-03T17:28:24.995Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.15658517Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131702024-02-03T17:28:24.995Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156657505Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131712024-02-03T17:28:24.995Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131722024-02-03T17:28:24.995Z,"time":"2024-02-03T17:28:24.15672919Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131732024-02-03T17:28:24.995Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156798461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131742024-02-03T17:28:24.995Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156828767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131752024-02-03T17:28:24.995Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156850928Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131762024-02-03T17:28:24.996Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156872717Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131772024-02-03T17:28:24.996Z{"msg":"5bf63fc5-e8ef-4df1-9ccf-f14fbc6d4355 is now active with session: 958ce1d2-af4f-44b6-8c3b-28e3664c24c8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.15689413Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131782024-02-03T17:28:24.996Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.15693328Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623,"session_id":"958ce1d2-af4f-44b6-8c3b-28e3664c24c8"}
131792024-02-03T17:28:24.996Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:24.156955521Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623}
131802024-02-03T17:28:24.996ZLoading write count information from file "/tmp/test_repair_verify.data"
131812024-02-03T17:28:24.996ZRun Repair workload
131822024-02-03T17:28:24.996Z01/30 Write block 562 len 32768 data: 4 7 8 10 12 13 10 10
131832024-02-03T17:28:24.996Z02/30 Write block 132 len 8192 data: 13 10
131842024-02-03T17:28:25.000Z03/30 Read block 56 len 16384
131852024-02-03T17:28:25.000Z04/30 Flush
131862024-02-03T17:28:25.000Z05/30 Write block 252 len 28672 data: 7 6 9 9 8 5 6
131872024-02-03T17:28:25.000Z06/30 Write block 208 len 28672 data: 10 8 9 7 6 6 8
131882024-02-03T17:28:25.000Z07/30 Write block 573 len 40960 data: 6 6 8 10 11 14 14 13 13 12
131892024-02-03T17:28:25.000Z08/30 Flush
131902024-02-03T17:28:25.004Z09/30 Write block 288 len 20480 data: 9 8 8 9 9
131912024-02-03T17:28:25.007Z10/30 Write block 499 len 12288 data: 12 12 14
131922024-02-03T17:28:25.008Z11/30 Read block 253 len 20480
131932024-02-03T17:28:25.010Z12/30 Read block 115 len 28672
131942024-02-03T17:28:25.010Z13/30 Read block 535 len 36864
131952024-02-03T17:28:25.015Z14/30 Read block 543 len 12288
131962024-02-03T17:28:25.015Z15/30 Read block 195 len 32768
131972024-02-03T17:28:25.018Z16/30 Read block 202 len 12288
131982024-02-03T17:28:25.018Z17/30 Write block 367 len 4096 data: 8
131992024-02-03T17:28:25.018Z18/30 Read block 593 len 28672
132002024-02-03T17:28:25.018Z19/30 Read block 498 len 16384
132012024-02-03T17:28:25.021Z20/30 Write block 25 len 8192 data: 11 9
132022024-02-03T17:28:25.021Z21/30 Write block 344 len 8192 data: 7 7
132032024-02-03T17:28:25.021Z22/30 Write block 189 len 12288 data: 10 9 9
132042024-02-03T17:28:25.021Z23/30 Read block 33 len 40960
132052024-02-03T17:28:25.025Z24/30 Write block 502 len 8192 data: 14 14
132062024-02-03T17:28:25.025Z25/30 Read block 533 len 28672
132072024-02-03T17:28:25.028Z26/30 Write block 498 len 16384 data: 11 13 13 15
132082024-02-03T17:28:25.028Z27/30 Write block 548 len 40960 data: 12 13 12 11 9 9 9 9 9 8
132092024-02-03T17:28:25.031Z28/30 Write block 74 len 32768 data: 6 7 9 10 9 8 8 7
132102024-02-03T17:28:25.031Z29/30 Write block 277 len 4096 data: 14
132112024-02-03T17:28:25.034Z30/30 Write block 128 len 40960 data: 12 14 15 15 14 11 14 16 18 19
132122024-02-03T17:28:25.034Z----------------------------------------------------------------
132132024-02-03T17:28:25.034Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:26
132142024-02-03T17:28:25.034ZStates: Active Active Active
132152024-02-03T17:28:25.034ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132162024-02-03T17:28:25.034Z 5 Acked 1004 Write 7 Done Done Sent false
132172024-02-03T17:28:25.034Z 6 Acked 1005 Write 7 Done Done Sent false
132182024-02-03T17:28:25.034Z 7 Acked 1006 Write 10 Done Done Sent false
132192024-02-03T17:28:25.034Z 8 Acked 1007 Flush 0 Done Done Sent false
132202024-02-03T17:28:25.034Z 9 Acked 1008 Write 5 Done Done Sent false
132212024-02-03T17:28:25.035Z 10 Acked 1009 Write 3 Done Done Sent false
132222024-02-03T17:28:25.035Z 11 Acked 1010 Read 5 Done Done Sent false
132232024-02-03T17:28:25.035Z 12 Acked 1011 Read 7 Done Done Sent false
132242024-02-03T17:28:25.035Z 13 Acked 1012 Read 9 Done Done Sent false
132252024-02-03T17:28:25.035Z 14 Acked 1013 Read 3 Done Done Sent false
132262024-02-03T17:28:25.035Z 15 Acked 1014 Read 8 Done Done Sent false
132272024-02-03T17:28:25.035Z 16 Acked 1015 Read 3 Done Done Sent false
132282024-02-03T17:28:25.035Z 17 Acked 1016 Write 1 Done Done Sent false
132292024-02-03T17:28:25.035Z 18 Acked 1017 Read 7 Done Done Sent false
132302024-02-03T17:28:25.035Z 19 Acked 1018 Read 4 Done Done Sent false
132312024-02-03T17:28:25.035Z 20 Acked 1019 Write 2 Done Done Sent false
132322024-02-03T17:28:25.035Z 21 Acked 1020 Write 2 Done Done Sent false
132332024-02-03T17:28:25.035Z 22 Acked 1021 Write 3 Done Done Sent false
132342024-02-03T17:28:25.035Z 23 Acked 1022 Read 10 Done Done Sent false
132352024-02-03T17:28:25.035Z 24 Acked 1023 Write 2 Done Done Sent false
132362024-02-03T17:28:25.035Z 25 Acked 1024 Read 7 Done Done Sent false
132372024-02-03T17:28:25.035Z 26 Acked 1025 Write 4 Done Done Sent false
132382024-02-03T17:28:25.035Z 27 Acked 1026 Write 10 Done Done Sent false
132392024-02-03T17:28:25.035Z 28 Acked 1027 Write 8 Done Done Sent false
132402024-02-03T17:28:25.035Z 29 Acked 1028 Write 1 Done Done Sent false
132412024-02-03T17:28:25.035Z 30 Acked 1029 Write 10 Sent Sent Sent false
132422024-02-03T17:28:25.035Z STATES DS:0 DS:1 DS:2 TOTAL
132432024-02-03T17:28:25.035Z New 0 0 0 0
132442024-02-03T17:28:25.035Z Sent 1 1 26 28
132452024-02-03T17:28:25.035Z Done 25 25 0 50
132462024-02-03T17:28:25.036Z Skipped 0 0 0 0
132472024-02-03T17:28:25.036Z Error 0 0 0 0
132482024-02-03T17:28:25.036ZLast Flush: 1007 1007 1003
132492024-02-03T17:28:25.036ZDownstairs last five completed: 1003 1002 1001 1000
132502024-02-03T17:28:25.036ZUpstairs last five completed: 30 29 28 27 26
132512024-02-03T17:28:25.036Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:24.203020514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1623}
132522024-02-03T17:28:25.036ZWrote out file "/tmp/test_repair_verify.data"
132532024-02-03T17:28:25.040Z
132542024-02-03T17:28:25.048ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132552024-02-03T17:28:25.082ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132562024-02-03T17:28:25.082Z 0 000-019 117 117 117 212 212 212 F F F
132572024-02-03T17:28:25.082Z 1 020-039 121 121 121 221 221 221 T T F <---
132582024-02-03T17:28:25.082Z 2 040-059 116 116 116 211 211 211 F F F
132592024-02-03T17:28:25.083Z 3 060-079 116 116 116 211 211 211 T T F <---
132602024-02-03T17:28:25.083Z 4 080-099 121 121 121 221 221 221 T T F <---
132612024-02-03T17:28:25.083Z 5 100-119 119 119 119 217 217 217 F F F
132622024-02-03T17:28:25.083Z 6 120-139 122 122 122 222 222 222 T T F <---
132632024-02-03T17:28:25.083Z 7 140-159 121 121 121 221 221 221 F F F
132642024-02-03T17:28:25.083Z 8 160-179 121 121 121 221 221 221 F F F
132652024-02-03T17:28:25.086Z 9 180-199 116 116 116 211 211 211 T T F <---
132662024-02-03T17:28:25.086Z 10 200-219 122 122 121 223 223 221 F F F <---
132672024-02-03T17:28:25.086Z 11 220-239 120 120 120 220 220 220 F F F
132682024-02-03T17:28:25.086Z 12 240-259 122 122 121 223 223 221 F F F <---
132692024-02-03T17:28:25.086Z 13 260-279 120 120 120 220 220 220 T T F <---
132702024-02-03T17:28:25.086Z 14 280-299 120 120 120 220 220 220 T T F <---
132712024-02-03T17:28:25.086Z 15 300-319 120 120 120 220 220 220 F F F
132722024-02-03T17:28:25.086Z 16 320-339 119 119 119 217 217 217 F F F
132732024-02-03T17:28:25.086Z 17 340-359 120 120 120 220 220 220 T T F <---
132742024-02-03T17:28:25.086Z 18 360-379 120 120 120 219 219 219 T T F <---
132752024-02-03T17:28:25.086Z 19 380-399 116 116 116 208 208 208 F F F
132762024-02-03T17:28:25.086Z 20 400-419 118 118 118 215 215 215 F F F
132772024-02-03T17:28:25.086Z 21 420-439 121 121 121 221 221 221 F F F
132782024-02-03T17:28:25.086Z 22 440-459 119 119 119 217 217 217 F F F
132792024-02-03T17:28:25.086Z 23 460-479 104 104 104 185 185 185 F F F
132802024-02-03T17:28:25.086Z 24 480-499 121 121 121 221 221 221 T T F <---
132812024-02-03T17:28:25.086Z 25 500-519 119 119 119 217 217 217 T T F <---
132822024-02-03T17:28:25.086Z 26 520-539 120 120 120 218 218 218 F F F
132832024-02-03T17:28:25.087Z 27 540-559 120 120 120 218 218 218 T T F <---
132842024-02-03T17:28:25.087Z 28 560-579 122 122 122 223 223 222 F F F <---
132852024-02-03T17:28:25.087Z 29 580-599 122 122 120 223 223 220 F F F <---
132862024-02-03T17:28:25.087ZMax gen: 122, Max flush: 223
132872024-02-03T17:28:25.087ZError: Difference in extent metadata found!
132882024-02-03T17:28:25.087ZA Difference in extent metadata is expected here
132892024-02-03T17:28:25.087ZOn loop 61
132902024-02-03T17:28:25.087Z
132912024-02-03T17:28:25.090ZVerifying data now
132922024-02-03T17:28:30.486ZNo change after verify
132932024-02-03T17:28:30.486ZLoop: 61 Downstairs dump after verify (and repair):
132942024-02-03T17:28:30.510ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132952024-02-03T17:28:30.511Z 0 000-019 117 117 117 212 212 212 F F F
132962024-02-03T17:28:30.511Z 1 020-039 123 123 123 224 224 224 F F F
132972024-02-03T17:28:30.511Z 2 040-059 116 116 116 211 211 211 F F F
132982024-02-03T17:28:30.511Z 3 060-079 123 123 123 224 224 224 F F F
132992024-02-03T17:28:30.511Z 4 080-099 123 123 123 224 224 224 F F F
133002024-02-03T17:28:30.511Z 5 100-119 119 119 119 217 217 217 F F F
133012024-02-03T17:28:30.511Z 6 120-139 123 123 123 224 224 224 F F F
133022024-02-03T17:28:30.515Z 7 140-159 121 121 121 221 221 221 F F F
133032024-02-03T17:28:30.515Z 8 160-179 121 121 121 221 221 221 F F F
133042024-02-03T17:28:30.515Z 9 180-199 123 123 123 224 224 224 F F F
133052024-02-03T17:28:30.515Z 10 200-219 122 122 122 223 223 223 F F F
133062024-02-03T17:28:30.515Z 11 220-239 120 120 120 220 220 220 F F F
133072024-02-03T17:28:30.516Z 12 240-259 122 122 122 223 223 223 F F F
133082024-02-03T17:28:30.516Z 13 260-279 123 123 123 224 224 224 F F F
133092024-02-03T17:28:30.516Z 14 280-299 123 123 123 224 224 224 F F F
133102024-02-03T17:28:30.516Z 15 300-319 120 120 120 220 220 220 F F F
133112024-02-03T17:28:30.516Z 16 320-339 119 119 119 217 217 217 F F F
133122024-02-03T17:28:30.516Z 17 340-359 123 123 123 224 224 224 F F F
133132024-02-03T17:28:30.516Z 18 360-379 123 123 123 224 224 224 F F F
133142024-02-03T17:28:30.516Z 19 380-399 116 116 116 208 208 208 F F F
133152024-02-03T17:28:30.516Z 20 400-419 118 118 118 215 215 215 F F F
133162024-02-03T17:28:30.516Z 21 420-439 121 121 121 221 221 221 F F F
133172024-02-03T17:28:30.516Z 22 440-459 119 119 119 217 217 217 F F F
133182024-02-03T17:28:30.516Z 23 460-479 104 104 104 185 185 185 F F F
133192024-02-03T17:28:30.516Z 24 480-499 123 123 123 224 224 224 F F F
133202024-02-03T17:28:30.516Z 25 500-519 123 123 123 224 224 224 F F F
133212024-02-03T17:28:30.517Z 26 520-539 120 120 120 218 218 218 F F F
133222024-02-03T17:28:30.517Z 27 540-559 123 123 123 224 224 224 F F F
133232024-02-03T17:28:30.517Z 28 560-579 122 122 122 223 223 223 F F F
133242024-02-03T17:28:30.517Z 29 580-599 122 122 122 223 223 223 F F F
133252024-02-03T17:28:30.517ZMax gen: 123, Max flush: 224
133262024-02-03T17:28:30.517Z
133272024-02-03T17:28:30.517ZBegin loop 62
133282024-02-03T17:28:30.517ZDownstairs to restart: 1
133292024-02-03T17:28:30.551Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.721047114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631}
133302024-02-03T17:28:30.551Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.721534878Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631}
133312024-02-03T17:28:30.551Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.721575938Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631}
133322024-02-03T17:28:30.676Z{"msg":"Crucible 9abf3e5e-89c8-44fb-8e19-80aab08217db has session id: a3e2038d-eeba-40ea-b646-f519344e5afc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.721646177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133332024-02-03T17:28:30.676Z{Crucible runtime is spawned
133342024-02-03T17:28:30.676Z"msg":"Upstairs opts: Upstairs UUID: 9abf3e5e-89c8-44fb-8e19-80aab08217db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.721827366Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133352024-02-03T17:28:30.677Z{"msg":"Crucible stats registered with UUID: 9abf3e5e-89c8-44fb-8e19-80aab08217db","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.721916918Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133362024-02-03T17:28:30.677Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.721958997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631}
133372024-02-03T17:28:30.677Z{"msg":"9abf3e5e-89c8-44fb-8e19-80aab08217db active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722019183Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133382024-02-03T17:28:30.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722057373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133392024-02-03T17:28:30.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722126859Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133402024-02-03T17:28:30.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.72216757Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133412024-02-03T17:28:30.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722241677Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133422024-02-03T17:28:30.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722280005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133432024-02-03T17:28:30.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.72231836Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133442024-02-03T17:28:30.677Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722355588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133452024-02-03T17:28:30.677Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722427746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133462024-02-03T17:28:30.677Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722566045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133472024-02-03T17:28:30.677Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722811822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133482024-02-03T17:28:30.677Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722903161Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133492024-02-03T17:28:30.678Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.722950382Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133502024-02-03T17:28:30.678Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.724398596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133512024-02-03T17:28:30.678Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.72445908Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133522024-02-03T17:28:30.678Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.724532122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133532024-02-03T17:28:30.678Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.724596911Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133542024-02-03T17:28:30.678Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.725226301Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133552024-02-03T17:28:30.678Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.725430369Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133562024-02-03T17:28:30.678Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.725561319Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133572024-02-03T17:28:30.678Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.725657923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133582024-02-03T17:28:30.678Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.726136042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133592024-02-03T17:28:30.678Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.726200231Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133602024-02-03T17:28:30.678Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.726259855Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133612024-02-03T17:28:30.678Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.72641643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133622024-02-03T17:28:30.678Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.726495654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133632024-02-03T17:28:30.678Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.726530866Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133642024-02-03T17:28:30.678Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.726650388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133652024-02-03T17:28:30.678Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.727421725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133662024-02-03T17:28:30.678Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.727505607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133672024-02-03T17:28:30.678Z{"msg":"[0]R flush_numbers[0..12]: [212, 224, 211, 224, 224, 217, 224, 221, 221, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.727580012Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133682024-02-03T17:28:30.678Z{"msg":"[0]R generation[0..12]: [117, 123, 116, 123, 123, 119, 123, 121, 121, 123, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.727654328Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133692024-02-03T17:28:30.679Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.727726851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133702024-02-03T17:28:30.679Z{"msg":"[1]R flush_numbers[0..12]: [212, 224, 211, 224, 224, 217, 224, 221, 221, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.727806645Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133712024-02-03T17:28:30.679Z{"msg":"[1]R generation[0..12]: [117, 123, 116, 123, 123, 119, 123, 121, 121, 123, 122, 120]","Wait for a query_work_queue command to finish before sending IO
133722024-02-03T17:28:30.679Zv":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.727966639Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133732024-02-03T17:28:30.679Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728065032Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133742024-02-03T17:28:30.679Z{"msg":"[2]R flush_numbers[0..12]: [212, 224, 211, 224, 224, 217, 224, 221, 221, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728132188Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133752024-02-03T17:28:30.679Z{"msg":"[2]R generation[0..12]: [117, 123, 116, 123, 123, 119, 123, 121, 121, 123, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728160728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133762024-02-03T17:28:30.679Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.72818656Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133772024-02-03T17:28:30.679Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.7282289Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133782024-02-03T17:28:30.679Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728253688Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133792024-02-03T17:28:30.679Z{"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728293885Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133802024-02-03T17:28:30.679Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728317887Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133812024-02-03T17:28:30.679Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728366151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133822024-02-03T17:28:30.679Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728388279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133832024-02-03T17:28:30.679Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728410079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133842024-02-03T17:28:30.679Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728432371Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133852024-02-03T17:28:30.679Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728456242Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133862024-02-03T17:28:30.679Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728490996Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133872024-02-03T17:28:30.679Z{"msg":"9abf3e5e-89c8-44fb-8e19-80aab08217db is now active with session: a3e2038d-eeba-40ea-b646-f519344e5afc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728556277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133882024-02-03T17:28:30.679Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728596637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631,"session_id":"a3e2038d-eeba-40ea-b646-f519344e5afc"}
133892024-02-03T17:28:30.679Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:29.728659289Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631}
133902024-02-03T17:28:30.679ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133912024-02-03T17:28:30.679ZLoading write count information from file "/tmp/test_repair_verify.data"
133922024-02-03T17:28:30.680ZRun Repair workload
133932024-02-03T17:28:30.680Z01/30 Write block 548 len 8192 data: 13 14
133942024-02-03T17:28:30.680Z02/30 Read block 164 len 8192
133952024-02-03T17:28:30.686Z03/30 Write block 504 len 8192 data: 14 18
133962024-02-03T17:28:30.686Z04/30 Read block 404 len 36864
133972024-02-03T17:28:30.686Z05/30 Read block 178 len 16384
133982024-02-03T17:28:30.686Z06/30 Write block 348 len 32768 data: 10 9 9 9 7 9 9 12
133992024-02-03T17:28:30.686Z07/30 Read block 84 len 20480
134002024-02-03T17:28:30.686Z08/30 Read block 309 len 32768
134012024-02-03T17:28:30.686Z09/30 Read block 359 len 28672
134022024-02-03T17:28:30.686Z10/30 Read block 579 len 24576
134032024-02-03T17:28:30.686Z11/30 Write block 316 len 32768 data: 8 7 6 10 8 8 9 8
134042024-02-03T17:28:30.686Z12/30 Write block 495 len 24576 data: 9 10 10 12 14 14
134052024-02-03T17:28:30.686Z13/30 Write block 291 len 12288 data: 10 10 11
134062024-02-03T17:28:30.686Z14/30 Write block 586 len 32768 data: 11 8 8 11 12 12 10 10
134072024-02-03T17:28:30.686Z15/30 Read block 178 len 36864
134082024-02-03T17:28:30.687Z16/30 Read block 162 len 20480
134092024-02-03T17:28:30.687Z17/30 Write block 537 len 8192 data: 9 8
134102024-02-03T17:28:30.687Z18/30 Read block 442 len 12288
134112024-02-03T17:28:30.687Z19/30 Read block 257 len 28672
134122024-02-03T17:28:30.687Z20/30 Read block 318 len 24576
134132024-02-03T17:28:30.687Z21/30 Write block 2 len 20480 data: 5 6 7 9 9
134142024-02-03T17:28:30.687Z22/30 Read block 267 len 20480
134152024-02-03T17:28:30.687Z23/30 Read block 282 len 20480
134162024-02-03T17:28:30.687Z24/30 Read block 435 len 8192
134172024-02-03T17:28:30.687Z25/30 Write block 169 len 40960 data: 10 10 9 8 8 9 10 9 12 11
134182024-02-03T17:28:30.687Z26/30 Flush
134192024-02-03T17:28:30.687Z27/30 Write block 112 len 12288 data: 12 14 16
134202024-02-03T17:28:30.687Z28/30 Write block 457 len 12288 data: 11 10 11
134212024-02-03T17:28:30.687Z29/30 Write block 99 len 20480 data: 7 8 8 8 9
134222024-02-03T17:28:30.687Z30/30 Write block 506 len 4096 data: 19
134232024-02-03T17:28:30.687Z----------------------------------------------------------------
134242024-02-03T17:28:30.688Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
134252024-02-03T17:28:30.688ZStates: Active Active Active
134262024-02-03T17:28:30.688ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134272024-02-03T17:28:30.688Z 1 Acked 1000 Write 2 Done Done Done false
134282024-02-03T17:28:30.688Z 2 Acked 1001 Read 2 Done Done Done false
134292024-02-03T17:28:30.688Z 3 Acked 1002 Write 2 Done Done Done false
134302024-02-03T17:28:30.688Z 4 Acked 1003 Read 9 Done Done Done false
134312024-02-03T17:28:30.688Z 5 Acked 1004 Read 4 Done Done Done false
134322024-02-03T17:28:30.688Z 6 Acked 1005 Write 8 Done Sent Done false
134332024-02-03T17:28:30.688Z 7 Acked 1006 Read 5 Done Sent Done false
134342024-02-03T17:28:30.688Z 8 Acked 1007 Read 8 Done Sent Done false
134352024-02-03T17:28:30.688Z 9 Acked 1008 Read 7 Done Sent Done false
134362024-02-03T17:28:30.688Z 10 Acked 1009 Read 6 Done Sent Done false
134372024-02-03T17:28:30.689Z 11 Acked 1010 Write 8 Done Sent Done false
134382024-02-03T17:28:30.689Z 12 Acked 1011 Write 6 Done Sent Done false
134392024-02-03T17:28:30.689Z 13 Acked 1012 Write 3 Done Sent Done false
134402024-02-03T17:28:30.689Z 14 Acked 1013 Write 8 Done Sent Done false
134412024-02-03T17:28:30.689Z 15 Acked 1014 Read 9 Done Sent Done false
134422024-02-03T17:28:30.689Z 16 Acked 1015 Read 5 Done Sent Done false
134432024-02-03T17:28:30.689Z 17 Acked 1016 Write 2 Done Sent Done false
134442024-02-03T17:28:30.689Z 18 Acked 1017 Read 3 Done Sent Done false
134452024-02-03T17:28:30.689Z 19 Acked 1018 Read 7 Done Sent Done false
134462024-02-03T17:28:30.689Z 20 Acked 1019 Read 6 Done Sent Done false
134472024-02-03T17:28:30.689Z 21 Acked 1020 Write 5 Done Sent Done false
134482024-02-03T17:28:30.689Z 22 Acked 1021 Read 5 Done Sent Done false
134492024-02-03T17:28:30.689Z 23 Acked 1022 Read 5 Done Sent Done false
134502024-02-03T17:28:30.690Z 24 Acked 1023 Read 2 Done Sent Done false
134512024-02-03T17:28:30.690Z 25 Acked 1024 Write 10 Done Sent Done false
134522024-02-03T17:28:30.690Z 26 Acked 1025 Flush 0 Done Sent Done false
134532024-02-03T17:28:30.690Z 27 Acked 1026 Write 3 Done Sent Done false
134542024-02-03T17:28:30.690Z 28 Acked 1027 Write 3 Done Sent Done false
134552024-02-03T17:28:30.690Z 29 Acked 1028 Write 5 Sent Sent Sent false
134562024-02-03T17:28:30.690Z 30 Acked 1029 Write 1 Sent Sent Sent false
134572024-02-03T17:28:30.690Z STATES DS:0 DS:1 DS:2 TOTAL
134582024-02-03T17:28:30.690Z New 0 0 0 0
134592024-02-03T17:28:30.690Z Sent 2 25 2 29
134602024-02-03T17:28:30.690Z Done 28 5 28 61
134612024-02-03T17:28:30.690Z Skipped 0 0 0 0
134622024-02-03T17:28:30.690Z Error 0 0 0 0
134632024-02-03T17:28:30.691ZLast Flush: 1025 0 1025
134642024-02-03T17:28:30.691ZDownstairs last five completed:
134652024-02-03T17:28:30.691ZUpstairs last five completed: 30 29 28 27 26
134662024-02-03T17:28:30.691Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:29.84760559Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1631}
134672024-02-03T17:28:30.691ZWrote out file "/tmp/test_repair_verify.data"
134682024-02-03T17:28:30.695Z
134692024-02-03T17:28:30.695ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134702024-02-03T17:28:30.724ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134712024-02-03T17:28:30.724Z 0 000-019 124 117 124 225 212 225 F F F <---
134722024-02-03T17:28:30.724Z 1 020-039 123 123 123 224 224 224 F F F
134732024-02-03T17:28:30.724Z 2 040-059 116 116 116 211 211 211 F F F
134742024-02-03T17:28:30.724Z 3 060-079 123 123 123 224 224 224 F F F
134752024-02-03T17:28:30.724Z 4 080-099 123 123 123 224 224 224 T F T <---
134762024-02-03T17:28:30.732Z 5 100-119 119 119 119 217 217 217 T F T <---
134772024-02-03T17:28:30.732Z 6 120-139 123 123 123 224 224 224 F F F
134782024-02-03T17:28:30.732Z 7 140-159 121 121 121 221 221 221 F F F
134792024-02-03T17:28:30.732Z 8 160-179 124 121 124 225 221 225 F F F <---
134802024-02-03T17:28:30.733Z 9 180-199 123 123 123 224 224 224 F F F
134812024-02-03T17:28:30.733Z 10 200-219 122 122 122 223 223 223 F F F
134822024-02-03T17:28:30.733Z 11 220-239 120 120 120 220 220 220 F F F
134832024-02-03T17:28:30.733Z 12 240-259 122 122 122 223 223 223 F F F
134842024-02-03T17:28:30.734Z 13 260-279 123 123 123 224 224 224 F F F
134852024-02-03T17:28:30.734Z 14 280-299 124 123 124 225 224 225 F F F <---
134862024-02-03T17:28:30.734Z 15 300-319 124 120 124 225 220 225 F F F <---
134872024-02-03T17:28:30.734Z 16 320-339 124 119 124 225 217 225 F F F <---
134882024-02-03T17:28:30.734Z 17 340-359 124 123 124 225 224 225 F F F <---
134892024-02-03T17:28:30.734Z 18 360-379 123 123 123 224 224 224 F F F
134902024-02-03T17:28:30.734Z 19 380-399 116 116 116 208 208 208 F F F
134912024-02-03T17:28:30.734Z 20 400-419 118 118 118 215 215 215 F F F
134922024-02-03T17:28:30.734Z 21 420-439 121 121 121 221 221 221 F F F
134932024-02-03T17:28:30.734Z 22 440-459 119 119 119 217 217 217 T F T <---
134942024-02-03T17:28:30.734Z 23 460-479 104 104 104 185 185 185 F F F
134952024-02-03T17:28:30.735Z 24 480-499 124 123 124 225 224 225 F F F <---
134962024-02-03T17:28:30.735Z 25 500-519 124 123 124 225 224 225 T T T <---
134972024-02-03T17:28:30.735Z 26 520-539 124 120 124 225 218 225 F F F <---
134982024-02-03T17:28:30.735Z 27 540-559 124 123 124 225 224 225 F T F <---
134992024-02-03T17:28:30.735Z 28 560-579 122 122 122 223 223 223 F F F
135002024-02-03T17:28:30.735Z 29 580-599 124 122 124 225 223 225 F F F <---
135012024-02-03T17:28:30.735ZMax gen: 124, Max flush: 225
135022024-02-03T17:28:30.735ZError: Difference in extent metadata found!
135032024-02-03T17:28:30.735ZA Difference in extent metadata is expected here
135042024-02-03T17:28:30.735ZOn loop 62
135052024-02-03T17:28:30.735Z
135062024-02-03T17:28:30.756ZVerifying data now
135072024-02-03T17:28:36.948ZNo change after verify
135082024-02-03T17:28:36.948ZLoop: 62 Downstairs dump after verify (and repair):
135092024-02-03T17:28:36.975ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135102024-02-03T17:28:36.975Z 0 000-019 124 124 124 225 225 225 F F F
135112024-02-03T17:28:36.979Z 1 020-039 123 123 123 224 224 224 F F F
135122024-02-03T17:28:36.979Z 2 040-059 116 116 116 211 211 211 F F F
135132024-02-03T17:28:36.979Z 3 060-079 123 123 123 224 224 224 F F F
135142024-02-03T17:28:36.979Z 4 080-099 125 125 125 226 226 226 F F F
135152024-02-03T17:28:36.979Z 5 100-119 125 125 125 226 226 226 F F F
135162024-02-03T17:28:36.979Z 6 120-139 123 123 123 224 224 224 F F F
135172024-02-03T17:28:36.979Z 7 140-159 121 121 121 221 221 221 F F F
135182024-02-03T17:28:36.979Z 8 160-179 124 124 124 225 225 225 F F F
135192024-02-03T17:28:36.979Z 9 180-199 123 123 123 224 224 224 F F F
135202024-02-03T17:28:36.979Z 10 200-219 122 122 122 223 223 223 F F F
135212024-02-03T17:28:36.979Z 11 220-239 120 120 120 220 220 220 F F F
135222024-02-03T17:28:36.979Z 12 240-259 122 122 122 223 223 223 F F F
135232024-02-03T17:28:36.980Z 13 260-279 123 123 123 224 224 224 F F F
135242024-02-03T17:28:36.980Z 14 280-299 124 124 124 225 225 225 F F F
135252024-02-03T17:28:36.980Z 15 300-319 124 124 124 225 225 225 F F F
135262024-02-03T17:28:36.980Z 16 320-339 124 124 124 225 225 225 F F F
135272024-02-03T17:28:36.980Z 17 340-359 124 124 124 225 225 225 F F F
135282024-02-03T17:28:36.980Z 18 360-379 123 123 123 224 224 224 F F F
135292024-02-03T17:28:36.980Z 19 380-399 116 116 116 208 208 208 F F F
135302024-02-03T17:28:36.980Z 20 400-419 118 118 118 215 215 215 F F F
135312024-02-03T17:28:36.980Z 21 420-439 121 121 121 221 221 221 F F F
135322024-02-03T17:28:36.980Z 22 440-459 125 125 125 226 226 226 F F F
135332024-02-03T17:28:36.980Z 23 460-479 104 104 104 185 185 185 F F F
135342024-02-03T17:28:36.980Z 24 480-499 124 124 124 225 225 225 F F F
135352024-02-03T17:28:36.980Z 25 500-519 125 125 125 226 226 226 F F F
135362024-02-03T17:28:36.980Z 26 520-539 124 124 124 225 225 225 F F F
135372024-02-03T17:28:36.980Z 27 540-559 124 124 124 225 225 225 F F F
135382024-02-03T17:28:36.980Z 28 560-579 122 122 122 223 223 223 F F F
135392024-02-03T17:28:36.980Z 29 580-599 124 124 124 225 225 225 F F F
135402024-02-03T17:28:36.980ZMax gen: 125, Max flush: 226
135412024-02-03T17:28:36.980Z
135422024-02-03T17:28:36.980ZBegin loop 63
135432024-02-03T17:28:36.980ZDownstairs to restart: 0
135442024-02-03T17:28:37.011Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.181270454Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639}
135452024-02-03T17:28:37.011Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.181781302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639}
135462024-02-03T17:28:37.016Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.181857171Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639}
135472024-02-03T17:28:37.016Z{"msg":"Crucible 83eac72b-f358-4b23-a0ea-3385a66a36b8 has session id: 809645cb-ba2d-4779-8b3e-8b23474b127a","v":0,"name":"crucible","level":30Crucible runtime is spawned
135482024-02-03T17:28:37.016Z,"time":"2024-02-03T17:28:36.181932435Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135492024-02-03T17:28:37.016Z{"msg":"Upstairs opts: Upstairs UUID: 83eac72b-f358-4b23-a0ea-3385a66a36b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.18204902Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135502024-02-03T17:28:37.016Z{"msg":"Crucible stats registered with UUID: 83eac72b-f358-4b23-a0ea-3385a66a36b8","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.18223572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135512024-02-03T17:28:37.016Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182322834Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639}
135522024-02-03T17:28:37.016Z{"msg":"83eac72b-f358-4b23-a0ea-3385a66a36b8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.18237524Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135532024-02-03T17:28:37.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182399793Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135542024-02-03T17:28:37.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182464988Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135552024-02-03T17:28:37.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182492716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135562024-02-03T17:28:37.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182526466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135572024-02-03T17:28:37.016Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182617816Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135582024-02-03T17:28:37.016Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182662305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135592024-02-03T17:28:37.016Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182727826Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135602024-02-03T17:28:37.016Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182773438Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135612024-02-03T17:28:37.016Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182857954Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135622024-02-03T17:28:37.016Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.182941156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135632024-02-03T17:28:37.016Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.18302966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135642024-02-03T17:28:37.016Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.183274445Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135652024-02-03T17:28:37.016Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.184154424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135662024-02-03T17:28:37.016Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.184236689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135672024-02-03T17:28:37.017Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.18428504Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135682024-02-03T17:28:37.017Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.184333758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135692024-02-03T17:28:37.017Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.185163439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135702024-02-03T17:28:37.017Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.185398702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135712024-02-03T17:28:37.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.18627753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135722024-02-03T17:28:37.022Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.186357042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135732024-02-03T17:28:37.022Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.186475784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135742024-02-03T17:28:37.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.186549057Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135752024-02-03T17:28:37.022Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.186578393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135762024-02-03T17:28:37.022Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.186621125Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135772024-02-03T17:28:37.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.186643695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135782024-02-03T17:28:37.022Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.186684533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135792024-02-03T17:28:37.022Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.187263778Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135802024-02-03T17:28:37.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.187971174Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135812024-02-03T17:28:37.022Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188049015Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135822024-02-03T17:28:37.022Z{"msg":"[0]R flush_numbers[0..12]: [225, 224, 211, 224, 226, 226, 224, 221, 225, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188077941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135832024-02-03T17:28:37.022Z{"msg":"[0]R generation[0..12]: [124, 123, 116, 123, 125, 125, 123, 121, 124, 123, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188101785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135842024-02-03T17:28:37.022Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188125549Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135852024-02-03T17:28:37.022Z{"msg":"[1]R flush_numbers[0..12]: [225, 224, 211, 224, 226, 226, 224, 221, 225, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.18814898Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135862024-02-03T17:28:37.022Z{"msg":"[1]R generation[0..12]: [124, 123, 116, 123, 125, 125, 123, 121, 124, 123, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188192194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135872024-02-03T17:28:37.022Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188218288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135882024-02-03T17:28:37.022Z{"msg":"[2]R flush_numbers[0..12]: [225, 224, 211, 224, 226, 226, 224, 221, 225, 224, 223, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188261873Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135892024-02-03T17:28:37.022Z{"msg":"[2]R generation[0..12]: [124, 123, 116, 123, 125, 125, 123, 121, 124, 123, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188287199Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135902024-02-03T17:28:37.023Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188343411Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135912024-02-03T17:28:37.023Z{"msg":"Max found gen is 126"Wait for a query_work_queue command to finish before sending IO
135922024-02-03T17:28:37.023Z,"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.18837093Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135932024-02-03T17:28:37.023Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188393876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135942024-02-03T17:28:37.023Z{"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188414934Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135952024-02-03T17:28:37.023Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188436652Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135962024-02-03T17:28:37.023Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188487378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135972024-02-03T17:28:37.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188509799Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135982024-02-03T17:28:37.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188531494Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
135992024-02-03T17:28:37.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188553613Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
136002024-02-03T17:28:37.023Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188575393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
136012024-02-03T17:28:37.023Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188596422Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
136022024-02-03T17:28:37.023Z{"msg":"83eac72b-f358-4b23-a0ea-3385a66a36b8 is now active with session: 809645cb-ba2d-4779-8b3e-8b23474b127a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188617595Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
136032024-02-03T17:28:37.023Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188637498Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639,"session_id":"809645cb-ba2d-4779-8b3e-8b23474b127a"}
136042024-02-03T17:28:37.023Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:36.188684252Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639}
136052024-02-03T17:28:37.023ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
136062024-02-03T17:28:37.023ZLoading write count information from file "/tmp/test_repair_verify.data"
136072024-02-03T17:28:37.023ZRun Repair workload
136082024-02-03T17:28:37.023Z01/30 Flush
136092024-02-03T17:28:37.023Z02/30 Flush
136102024-02-03T17:28:37.030Z03/30 Flush
136112024-02-03T17:28:37.030Z04/30 Write block 503 len 24576 data: 15 15 19 20 20 17
136122024-02-03T17:28:37.031Z05/30 Write block 207 len 28672 data: 10 11 9 10 8 7 7
136132024-02-03T17:28:37.031Z06/30 Flush
136142024-02-03T17:28:37.036Z07/30 Write block 424 len 8192 data: 11 9
136152024-02-03T17:28:37.036Z08/30 Write block 330 len 16384 data: 8 8 8 9
136162024-02-03T17:28:37.036Z09/30 Read block 158 len 8192
136172024-02-03T17:28:37.036Z10/30 Read block 548 len 28672
136182024-02-03T17:28:37.037Z11/30 Read block 80 len 28672
136192024-02-03T17:28:37.037Z12/30 Write block 202 len 12288 data: 7 9 8
136202024-02-03T17:28:37.037Z13/30 Read block 138 len 8192
136212024-02-03T17:28:37.040Z14/30 Read block 348 len 4096
136222024-02-03T17:28:37.040Z15/30 Read block 268 len 20480
136232024-02-03T17:28:37.040Z16/30 Read block 85 len 4096
136242024-02-03T17:28:37.044Z17/30 Read block 580 len 20480
136252024-02-03T17:28:37.044Z18/30 Read block 357 len 24576
136262024-02-03T17:28:37.044Z19/30 Read block 251 len 20480
136272024-02-03T17:28:37.049Z20/30 Write block 83 len 28672 data: 12 13 10 10 9 7 10
136282024-02-03T17:28:37.049Z21/30 Read block 593 len 12288
136292024-02-03T17:28:37.049Z22/30 Read block 51 len 8192
136302024-02-03T17:28:37.049Z23/30 Read block 534 len 24576
136312024-02-03T17:28:37.052Z24/30 Read block 80 len 36864
136322024-02-03T17:28:37.055Z25/30 Write block 294 len 20480 data: 11 9 10 11 7
136332024-02-03T17:28:37.055Z26/30 Read block 295 len 40960
136342024-02-03T17:28:37.068Z27/30 Write block 274 len 20480 data: 15 16 16 15 15
136352024-02-03T17:28:37.068Z28/30 Write block 2 len 36864 data: 6 7 8 10 10 11 10 10 14
136362024-02-03T17:28:37.068Z29/30 Write block 125 len 32768 data: 7 10 10 13 15 16 16 15
136372024-02-03T17:28:37.074Z30/30 Read block 271 len 16384
136382024-02-03T17:28:37.074Z----------------------------------------------------------------
136392024-02-03T17:28:37.074Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
136402024-02-03T17:28:37.074ZStates: Active Active Active
136412024-02-03T17:28:37.074ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136422024-02-03T17:28:37.075Z 1 Acked 1000 Flush 0 Sent Done Done false
136432024-02-03T17:28:37.075Z 2 Acked 1001 Flush 0 Sent Done Done false
136442024-02-03T17:28:37.075Z 3 Acked 1002 Flush 0 Sent Done Done false
136452024-02-03T17:28:37.075Z 4 Acked 1003 Write 6 Sent Done Done false
136462024-02-03T17:28:37.075Z 5 Acked 1004 Write 7 Sent Done Done false
136472024-02-03T17:28:37.075Z 6 Acked 1005 Flush 0 Sent Done Done false
136482024-02-03T17:28:37.075Z 7 Acked 1006 Write 2 Sent Done Done false
136492024-02-03T17:28:37.075Z 8 Acked 1007 Write 4 Sent Done Done false
136502024-02-03T17:28:37.075Z 9 Acked 1008 Read 2 Sent Done Done false
136512024-02-03T17:28:37.075Z 10 Acked 1009 Read 7 Sent Done Done false
136522024-02-03T17:28:37.075Z 11 Acked 1010 Read 7 Sent Done Done false
136532024-02-03T17:28:37.075Z 12 Acked 1011 Write 3 Sent Done Done false
136542024-02-03T17:28:37.075Z 13 Acked 1012 Read 2 Sent Done Done false
136552024-02-03T17:28:37.075Z 14 Acked 1013 Read 1 Sent Done Done false
136562024-02-03T17:28:37.075Z 15 Acked 1014 Read 5 Sent Done Done false
136572024-02-03T17:28:37.075Z 16 Acked 1015 Read 1 Sent Done Done false
136582024-02-03T17:28:37.075Z 17 Acked 1016 Read 5 Sent Done Done false
136592024-02-03T17:28:37.075Z 18 Acked 1017 Read 6 Sent Done Done false
136602024-02-03T17:28:37.075Z 19 Acked 1018 Read 5 Sent Done Done false
136612024-02-03T17:28:37.075Z 20 Acked 1019 Write 7 Sent Done Done false
136622024-02-03T17:28:37.075Z 21 Acked 1020 Read 3 Sent Done Done false
136632024-02-03T17:28:37.075Z 22 Acked 1021 Read 2 Sent Done Done false
136642024-02-03T17:28:37.075Z 23 Acked 1022 Read 6 Sent Done Done false
136652024-02-03T17:28:37.075Z 24 Acked 1023 Read 9 Sent Done Done false
136662024-02-03T17:28:37.076Z 25 Acked 1024 Write 5 Sent Done Done false
136672024-02-03T17:28:37.076Z 26 Acked 1025 Read 10 Sent Done Done false
136682024-02-03T17:28:37.076Z 27 Acked 1026 Write 5 Sent Done Done false
136692024-02-03T17:28:37.076Z 28 Acked 1027 Write 9 Sent Done Done false
136702024-02-03T17:28:37.076Z 29 Acked 1028 Write 8 Sent Done Done false
136712024-02-03T17:28:37.076Z 30 Acked 1029 Read 4 Sent Done Done false
136722024-02-03T17:28:37.076Z STATES DS:0 DS:1 DS:2 TOTAL
136732024-02-03T17:28:37.076Z New 0 0 0 0
136742024-02-03T17:28:37.076Z Sent 30 0 0 30
136752024-02-03T17:28:37.076Z Done 0 30 30 60
136762024-02-03T17:28:37.076Z Skipped 0 0 0 0
136772024-02-03T17:28:37.076Z Error 0 0 0 0
136782024-02-03T17:28:37.076ZLast Flush: 0 1005 1005
136792024-02-03T17:28:37.076ZDownstairs last five completed:
136802024-02-03T17:28:37.076ZUpstairs last five completed: 30 29 28 27 26
136812024-02-03T17:28:37.076Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:36.233031892Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1639}
136822024-02-03T17:28:37.076ZWrote out file "/tmp/test_repair_verify.data"
136832024-02-03T17:28:37.076Z
136842024-02-03T17:28:37.081ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136852024-02-03T17:28:37.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136862024-02-03T17:28:37.110Z 0 000-019 124 124 124 225 225 225 F T T <---
136872024-02-03T17:28:37.110Z 1 020-039 123 123 123 224 224 224 F F F
136882024-02-03T17:28:37.110Z 2 040-059 116 116 116 211 211 211 F F F
136892024-02-03T17:28:37.110Z 3 060-079 123 123 123 224 224 224 F F F
136902024-02-03T17:28:37.120Z 4 080-099 125 125 125 226 226 226 F T T <---
136912024-02-03T17:28:37.120Z 5 100-119 125 125 125 226 226 226 F F F
136922024-02-03T17:28:37.120Z 6 120-139 123 123 123 224 224 224 F T T <---
136932024-02-03T17:28:37.120Z 7 140-159 121 121 121 221 221 221 F F F
136942024-02-03T17:28:37.121Z 8 160-179 124 124 124 225 225 225 F F F
136952024-02-03T17:28:37.121Z 9 180-199 123 123 123 224 224 224 F F F
136962024-02-03T17:28:37.121Z 10 200-219 122 126 126 223 230 230 F T T <---
136972024-02-03T17:28:37.121Z 11 220-239 120 120 120 220 220 220 F F F
136982024-02-03T17:28:37.121Z 12 240-259 122 122 122 223 223 223 F F F
136992024-02-03T17:28:37.121Z 13 260-279 123 123 123 224 224 224 F T T <---
137002024-02-03T17:28:37.121Z 14 280-299 124 124 124 225 225 225 F T T <---
137012024-02-03T17:28:37.121Z 15 300-319 124 124 124 225 225 225 F F F
137022024-02-03T17:28:37.121Z 16 320-339 124 124 124 225 225 225 F T T <---
137032024-02-03T17:28:37.121Z 17 340-359 124 124 124 225 225 225 F F F
137042024-02-03T17:28:37.121Z 18 360-379 123 123 123 224 224 224 F F F
137052024-02-03T17:28:37.121Z 19 380-399 116 116 116 208 208 208 F F F
137062024-02-03T17:28:37.121Z 20 400-419 118 118 118 215 215 215 F F F
137072024-02-03T17:28:37.121Z 21 420-439 121 121 121 221 221 221 F T T <---
137082024-02-03T17:28:37.121Z 22 440-459 125 125 125 226 226 226 F F F
137092024-02-03T17:28:37.121Z 23 460-479 104 104 104 185 185 185 F F F
137102024-02-03T17:28:37.121Z 24 480-499 124 124 124 225 225 225 F F F
137112024-02-03T17:28:37.121Z 25 500-519 125 126 126 226 230 230 F F F <---
137122024-02-03T17:28:37.121Z 26 520-539 124 124 124 225 225 225 F F F
137132024-02-03T17:28:37.121Z 27 540-559 124 124 124 225 225 225 F F F
137142024-02-03T17:28:37.121Z 28 560-579 122 122 122 223 223 223 F F F
137152024-02-03T17:28:37.121Z 29 580-599 124 124 124 225 225 225 F F F
137162024-02-03T17:28:37.121ZMax gen: 126, Max flush: 230
137172024-02-03T17:28:37.122ZError: Difference in extent metadata found!
137182024-02-03T17:28:37.122ZA Difference in extent metadata is expected here
137192024-02-03T17:28:37.122ZOn loop 63
137202024-02-03T17:28:37.122Z
137212024-02-03T17:28:37.122ZVerifying data now
137222024-02-03T17:28:42.082ZNo change after verify
137232024-02-03T17:28:42.082ZLoop: 63 Downstairs dump after verify (and repair):
137242024-02-03T17:28:42.127ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137252024-02-03T17:28:42.127Z 0 000-019 127 127 127 231 231 231 F F F
137262024-02-03T17:28:42.127Z 1 020-039 123 123 123 224 224 224 F F F
137272024-02-03T17:28:42.127Z 2 040-059 116 116 116 211 211 211 F F F
137282024-02-03T17:28:42.127Z 3 060-079 123 123 123 224 224 224 F F F
137292024-02-03T17:28:42.127Z 4 080-099 127 127 127 231 231 231 F F F
137302024-02-03T17:28:42.127Z 5 100-119 125 125 125 226 226 226 F F F
137312024-02-03T17:28:42.127Z 6 120-139 127 127 127 231 231 231 F F F
137322024-02-03T17:28:42.127Z 7 140-159 121 121 121 221 221 221 F F F
137332024-02-03T17:28:42.127Z 8 160-179 124 124 124 225 225 225 F F F
137342024-02-03T17:28:42.127Z 9 180-199 123 123 123 224 224 224 F F F
137352024-02-03T17:28:42.128Z 10 200-219 127 127 127 231 231 231 F F F
137362024-02-03T17:28:42.128Z 11 220-239 120 120 120 220 220 220 F F F
137372024-02-03T17:28:42.128Z 12 240-259 122 122 122 223 223 223 F F F
137382024-02-03T17:28:42.128Z 13 260-279 127 127 127 231 231 231 F F F
137392024-02-03T17:28:42.128Z 14 280-299 127 127 127 231 231 231 F F F
137402024-02-03T17:28:42.128Z 15 300-319 124 124 124 225 225 225 F F F
137412024-02-03T17:28:42.128Z 16 320-339 127 127 127 231 231 231 F F F
137422024-02-03T17:28:42.128Z 17 340-359 124 124 124 225 225 225 F F F
137432024-02-03T17:28:42.128Z 18 360-379 123 123 123 224 224 224 F F F
137442024-02-03T17:28:42.128Z 19 380-399 116 116 116 208 208 208 F F F
137452024-02-03T17:28:42.128Z 20 400-419 118 118 118 215 215 215 F F F
137462024-02-03T17:28:42.128Z 21 420-439 127 127 127 231 231 231 F F F
137472024-02-03T17:28:42.128Z 22 440-459 125 125 125 226 226 226 F F F
137482024-02-03T17:28:42.128Z 23 460-479 104 104 104 185 185 185 F F F
137492024-02-03T17:28:42.128Z 24 480-499 124 124 124 225 225 225 F F F
137502024-02-03T17:28:42.128Z 25 500-519 126 126 126 230 230 230 F F F
137512024-02-03T17:28:42.128Z 26 520-539 124 124 124 225 225 225 F F F
137522024-02-03T17:28:42.128Z 27 540-559 124 124 124 225 225 225 F F F
137532024-02-03T17:28:42.128Z 28 560-579 122 122 122 223 223 223 F F F
137542024-02-03T17:28:42.128Z 29 580-599 124 124 124 225 225 225 F F F
137552024-02-03T17:28:42.128ZMax gen: 127, Max flush: 231
137562024-02-03T17:28:42.128Z
137572024-02-03T17:28:42.128ZBegin loop 64
137582024-02-03T17:28:42.129ZDownstairs to restart: 1
137592024-02-03T17:28:42.129Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.203710732Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647}
137602024-02-03T17:28:42.129Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204213246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647}
137612024-02-03T17:28:42.129Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204247334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647}
137622024-02-03T17:28:42.129Z{"msg":"Crucible ac00e10e-ae7d-466d-9bec-da61c26a1791 has session id: f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204315046Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137632024-02-03T17:28:42.129Z{"msg":"Upstairs opts: Upstairs UUID: ac00e10e-ae7d-466d-9bec-da61c26a1791, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"Crucible runtime is spawned
137642024-02-03T17:28:42.129Zname":"crucible","level":30,"time":"2024-02-03T17:28:41.204406754Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137652024-02-03T17:28:42.129Z{"msg":"Crucible stats registered with UUID: ac00e10e-ae7d-466d-9bec-da61c26a1791","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204468769Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137662024-02-03T17:28:42.129Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204493827Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647}
137672024-02-03T17:28:42.129Z{"msg":"ac00e10e-ae7d-466d-9bec-da61c26a1791 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204712422Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137682024-02-03T17:28:42.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204799215Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137692024-02-03T17:28:42.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204826274Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137702024-02-03T17:28:42.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204848612Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137712024-02-03T17:28:42.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204916538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137722024-02-03T17:28:42.129Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.204964099Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137732024-02-03T17:28:42.129Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.205007994Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137742024-02-03T17:28:42.129Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.205096432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137752024-02-03T17:28:42.129Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.205126892Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137762024-02-03T17:28:42.129Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.205164691Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137772024-02-03T17:28:42.129Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.205411374Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137782024-02-03T17:28:42.129Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.205506195Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137792024-02-03T17:28:42.129Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.205765792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137802024-02-03T17:28:42.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.207192287Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137812024-02-03T17:28:42.130Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.207268504Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137822024-02-03T17:28:42.130Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.207313522Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137832024-02-03T17:28:42.130Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.207378211Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137842024-02-03T17:28:42.130Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.207594923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137852024-02-03T17:28:42.130Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.207695674Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137862024-02-03T17:28:42.130Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.208484715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137872024-02-03T17:28:42.130Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.208586034Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137882024-02-03T17:28:42.130Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.208677622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137892024-02-03T17:28:42.130Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209467163Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137902024-02-03T17:28:42.130Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209539985Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137912024-02-03T17:28:42.130Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209567787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137922024-02-03T17:28:42.130Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209596985Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137932024-02-03T17:28:42.130Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209658773Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137942024-02-03T17:28:42.130Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209686294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137952024-02-03T17:28:42.130Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209726499Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137962024-02-03T17:28:42.130Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209750443Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137972024-02-03T17:28:42.130Z{"msg":"[0]R flush_numbers[0..12]: [231, 224, 211, 224, 231, 226, 231, 221, 225, 224, 231, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.20979305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137982024-02-03T17:28:42.130Z{"msg":"[0]R generation[0..12]: [127, 123, 116, 123, 127, 125, 127, 121, 124, 123, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209818877Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
137992024-02-03T17:28:42.130Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209861336Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138002024-02-03T17:28:42.130Z{"msg":"[1]R flush_numbers[0..12]: [231, 224, 211, 224, 231, 226, 231, 221, 225, 224, 231, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209886461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138012024-02-03T17:28:42.130Z{"msg":"[1]R generation[0..12]: [127, 123, 116, 123, 127, 125, 127, 121, 124, 123, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209929356Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138022024-02-03T17:28:42.130Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209954574Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138032024-02-03T17:28:42.130Z{"msg":"[2]R flush_numbers[0..12]: [231, 224, 211, 224, 231, 226, 231, 221, 225, 224, 231, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.209978935Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
138042024-02-03T17:28:42.131Zf7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138052024-02-03T17:28:42.131Z{"msg":"[2]R generation[0..12]: [127, 123, 116, 123, 127, 125, 127, 121, 124, 123, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210007722Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138062024-02-03T17:28:42.131Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210051799Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138072024-02-03T17:28:42.131Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210075575Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138082024-02-03T17:28:42.131Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210098923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138092024-02-03T17:28:42.131Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210139234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138102024-02-03T17:28:42.131Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.2101629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138112024-02-03T17:28:42.131Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.21018594Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138122024-02-03T17:28:42.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210225418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138132024-02-03T17:28:42.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210249138Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138142024-02-03T17:28:42.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210297466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138152024-02-03T17:28:42.131Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210321534Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138162024-02-03T17:28:42.131Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210362734Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138172024-02-03T17:28:42.131Z{"msg":"ac00e10e-ae7d-466d-9bec-da61c26a1791 is now active with session: f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210385812Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138182024-02-03T17:28:42.131Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210406852Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647,"session_id":"f7dce624-fbc6-4f4b-9765-43dcd3c8d6a3"}
138192024-02-03T17:28:42.131Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:41.210427502Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647}
138202024-02-03T17:28:42.131ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138212024-02-03T17:28:42.131ZLoading write count information from file "/tmp/test_repair_verify.data"
138222024-02-03T17:28:42.131ZRun Repair workload
138232024-02-03T17:28:42.131Z01/30 Write block 437 len 28672 data: 10 10 11 13 14 9 9
138242024-02-03T17:28:42.903Z02/30 Flush
138252024-02-03T17:28:42.904Z03/30 Write block 20 len 20480 data: 4 6 7 7 10
138262024-02-03T17:28:42.904Z04/30 Read block 146 len 40960
138272024-02-03T17:28:42.904Z05/30 Flush
138282024-02-03T17:28:42.904Z06/30 Write block 99 len 40960 data: 8 9 9 9 10 8 11 9 10 11
138292024-02-03T17:28:42.904Z07/30 Write block 558 len 8192 data: 9 10
138302024-02-03T17:28:42.904Z08/30 Flush
138312024-02-03T17:28:42.904Z09/30 Write block 437 len 12288 data: 11 11 12
138322024-02-03T17:28:42.904Z10/30 Read block 124 len 40960
138332024-02-03T17:28:42.904Z11/30 Write block 31 len 16384 data: 7 7 5 5
138342024-02-03T17:28:42.904Z12/30 Write block 139 len 24576 data: 18 14 13 11 6 6
138352024-02-03T17:28:42.904Z13/30 Read block 257 len 24576
138362024-02-03T17:28:42.904Z14/30 Read block 503 len 8192
138372024-02-03T17:28:42.904Z15/30 Write block 118 len 20480 data: 14 14 13 14 10
138382024-02-03T17:28:42.905Z16/30 Write block 439 len 8192 data: 13 14
138392024-02-03T17:28:42.905Z17/30 Read block 329 len 28672
138402024-02-03T17:28:42.905Z18/30 Read block 44 len 40960
138412024-02-03T17:28:42.905Z19/30 Read block 481 len 28672
138422024-02-03T17:28:42.905Z20/30 Read block 451 len 28672
138432024-02-03T17:28:42.905Z21/30 Read block 376 len 32768
138442024-02-03T17:28:42.905Z22/30 Write block 458 len 36864 data: 11 12 9 8 10 11 10 12 9
138452024-02-03T17:28:42.905Z23/30 Flush
138462024-02-03T17:28:42.905Z24/30 Write block 440 len 36864 data: 15 15 10 10 8 10 9 7 8
138472024-02-03T17:28:42.905Z25/30 Read block 493 len 20480
138482024-02-03T17:28:42.905Z26/30 Write block 253 len 36864 data: 7 10 10 9 6 7 5 7 9
138492024-02-03T17:28:42.905Z27/30 Read block 503 len 32768
138502024-02-03T17:28:42.905Z28/30 Read block 331 len 24576
138512024-02-03T17:28:42.905Z29/30 Write block 501 len 4096 data: 16
138522024-02-03T17:28:42.905Z30/30 Read block 300 len 32768
138532024-02-03T17:28:42.905Z----------------------------------------------------------------
138542024-02-03T17:28:42.905Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:22
138552024-02-03T17:28:42.906ZStates: Active Active Active
138562024-02-03T17:28:42.906ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138572024-02-03T17:28:42.906Z 9 Acked 1008 Write 3 Done Done Done false
138582024-02-03T17:28:42.906Z 10 Acked 1009 Read 10 Done Sent Done false
138592024-02-03T17:28:42.906Z 11 Acked 1010 Write 4 Done Sent Done false
138602024-02-03T17:28:42.906Z 12 Acked 1011 Write 6 Done Sent Done false
138612024-02-03T17:28:42.906Z 13 Acked 1012 Read 6 Done Sent Done false
138622024-02-03T17:28:42.906Z 14 Acked 1013 Read 2 Done Sent Done false
138632024-02-03T17:28:42.906Z 15 Acked 1014 Write 5 Done Sent Done false
138642024-02-03T17:28:42.906Z 16 Acked 1015 Write 2 Done Sent Done false
138652024-02-03T17:28:42.906Z 17 Acked 1016 Read 7 Done Sent Done false
138662024-02-03T17:28:42.906Z 18 Acked 1017 Read 10 Done Sent Done false
138672024-02-03T17:28:42.906Z 19 Acked 1018 Read 7 Done Sent Done false
138682024-02-03T17:28:42.907Z 20 Acked 1019 Read 7 Done Sent Done false
138692024-02-03T17:28:42.907Z 21 Acked 1020 Read 8 Done Sent Done false
138702024-02-03T17:28:42.907Z 22 Acked 1021 Write 9 Done Sent Done false
138712024-02-03T17:28:42.907Z 23 Acked 1022 Flush 0 Done Sent Done false
138722024-02-03T17:28:42.907Z 24 Acked 1023 Write 9 Done Sent Done false
138732024-02-03T17:28:42.907Z 25 Acked 1024 Read 5 Done Sent Done false
138742024-02-03T17:28:42.907Z 26 Acked 1025 Write 9 Done Sent Done false
138752024-02-03T17:28:42.907Z 27 Acked 1026 Read 8 Done Sent Done false
138762024-02-03T17:28:42.907Z 28 Acked 1027 Read 6 Done Sent Done false
138772024-02-03T17:28:42.907Z 29 Acked 1028 Write 1 Done Sent Done false
138782024-02-03T17:28:42.907Z 30 Acked 1029 Read 8 Done Sent Sent false
138792024-02-03T17:28:42.907Z STATES DS:0 DS:1 DS:2 TOTAL
138802024-02-03T17:28:42.907Z New 0 0 0 0
138812024-02-03T17:28:42.907Z Sent 0 21 1 22
138822024-02-03T17:28:42.907Z Done 22 1 21 44
138832024-02-03T17:28:42.907Z Skipped 0 0 0 0
138842024-02-03T17:28:42.908Z Error 0 0 0 0
138852024-02-03T17:28:42.908ZLast Flush: 1022 1007 1022
138862024-02-03T17:28:42.908ZDownstairs last five completed: 1007 1006 1005 1004 1003
138872024-02-03T17:28:42.908ZUpstairs last five completed: 30 29 28 27 26
138882024-02-03T17:28:42.908Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:41.266433755Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1647}
138892024-02-03T17:28:42.908ZWrote out file "/tmp/test_repair_verify.data"
138902024-02-03T17:28:42.908Z
138912024-02-03T17:28:42.908ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138922024-02-03T17:28:42.908ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138932024-02-03T17:28:42.908Z 0 000-019 127 127 127 231 231 231 F F F
138942024-02-03T17:28:42.908Z 1 020-039 128 128 128 235 233 235 F F F <---
138952024-02-03T17:28:42.908Z 2 040-059 116 116 116 211 211 211 F F F
138962024-02-03T17:28:42.908Z 3 060-079 123 123 123 224 224 224 F F F
138972024-02-03T17:28:42.908Z 4 080-099 128 128 128 234 234 234 F F F
138982024-02-03T17:28:42.909Z 5 100-119 128 128 128 235 234 235 F F F <---
138992024-02-03T17:28:42.909Z 6 120-139 128 127 128 235 231 235 F F F <---
139002024-02-03T17:28:42.909Z 7 140-159 128 121 128 235 221 235 F F F <---
139012024-02-03T17:28:42.909Z 8 160-179 124 124 124 225 225 225 F F F
139022024-02-03T17:28:42.909Z 9 180-199 123 123 123 224 224 224 F F F
139032024-02-03T17:28:42.909Z 10 200-219 127 127 127 231 231 231 F F F
139042024-02-03T17:28:42.909Z 11 220-239 120 120 120 220 220 220 F F F
139052024-02-03T17:28:42.909Z 12 240-259 122 122 122 223 223 223 T F T <---
139062024-02-03T17:28:42.909Z 13 260-279 127 127 127 231 231 231 T F T <---
139072024-02-03T17:28:42.909Z 14 280-299 127 127 127 231 231 231 F F F
139082024-02-03T17:28:42.909Z 15 300-319 124 124 124 225 225 225 F F F
139092024-02-03T17:28:42.909Z 16 320-339 127 127 127 231 231 231 F F F
139102024-02-03T17:28:42.909Z 17 340-359 124 124 124 225 225 225 F F F
139112024-02-03T17:28:42.909Z 18 360-379 123 123 123 224 224 224 F F F
139122024-02-03T17:28:42.909Z 19 380-399 116 116 116 208 208 208 F F F
139132024-02-03T17:28:42.909Z 20 400-419 118 118 118 215 215 215 F F F
139142024-02-03T17:28:42.909Z 21 420-439 128 128 128 235 232 235 F T F <---
139152024-02-03T17:28:42.910Z 22 440-459 128 128 128 235 232 235 T F T <---
139162024-02-03T17:28:42.910Z 23 460-479 128 104 128 235 185 235 F F F <---
139172024-02-03T17:28:42.910Z 24 480-499 124 124 124 225 225 225 F F F
139182024-02-03T17:28:42.910Z 25 500-519 126 126 126 230 230 230 T F T <---
139192024-02-03T17:28:42.910Z 26 520-539 124 124 124 225 225 225 F F F
139202024-02-03T17:28:42.910Z 27 540-559 128 128 128 234 234 234 F F F
139212024-02-03T17:28:42.910Z 28 560-579 122 122 122 223 223 223 F F F
139222024-02-03T17:28:42.910Z 29 580-599 124 124 124 225 225 225 F F F
139232024-02-03T17:28:42.910ZMax gen: 128, Max flush: 235
139242024-02-03T17:28:52.731ZError: Difference in extent metadata found!
139252024-02-03T17:28:52.731ZA Difference in extent metadata is expected here
139262024-02-03T17:28:52.731ZOn loop 64
139272024-02-03T17:28:52.731Z
139282024-02-03T17:28:52.731ZVerifying data now
139292024-02-03T17:28:53.111ZNo change after verify
139302024-02-03T17:28:53.111ZLoop: 64 Downstairs dump after verify (and repair):
139312024-02-03T17:28:53.112ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139322024-02-03T17:28:53.112Z 0 000-019 127 127 127 231 231 231 F F F
139332024-02-03T17:28:53.112Z 1 020-039 128 128 128 235 235 235 F F F
139342024-02-03T17:28:53.112Z 2 040-059 116 116 116 211 211 211 F F F
139352024-02-03T17:28:53.112Z 3 060-079 123 123 123 224 224 224 F F F
139362024-02-03T17:28:53.112Z 4 080-099 128 128 128 234 234 234 F F F
139372024-02-03T17:28:53.112Z 5 100-119 128 128 128 235 235 235 F F F
139382024-02-03T17:28:53.112Z 6 120-139 128 128 128 235 235 235 F F F
139392024-02-03T17:28:53.112Z 7 140-159 128 128 128 235 235 235 F F F
139402024-02-03T17:28:53.112Z 8 160-179 124 124 124 225 225 225 F F F
139412024-02-03T17:28:53.113Z 9 180-199 123 123 123 224 224 224 F F F
139422024-02-03T17:28:53.113Z 10 200-219 127 127 127 231 231 231 F F F
139432024-02-03T17:28:53.113Z 11 220-239 120 120 120 220 220 220 F F F
139442024-02-03T17:28:53.113Z 12 240-259 129 129 129 236 236 236 F F F
139452024-02-03T17:28:53.113Z 13 260-279 129 129 129 236 236 236 F F F
139462024-02-03T17:28:53.113Z 14 280-299 127 127 127 231 231 231 F F F
139472024-02-03T17:28:53.113Z 15 300-319 124 124 124 225 225 225 F F F
139482024-02-03T17:28:53.113Z 16 320-339 127 127 127 231 231 231 F F F
139492024-02-03T17:28:53.113Z 17 340-359 124 124 124 225 225 225 F F F
139502024-02-03T17:28:53.113Z 18 360-379 123 123 123 224 224 224 F F F
139512024-02-03T17:28:53.113Z 19 380-399 116 116 116 208 208 208 F F F
139522024-02-03T17:28:53.114Z 20 400-419 118 118 118 215 215 215 F F F
139532024-02-03T17:28:53.114Z 21 420-439 128 128 128 235 235 235 F F F
139542024-02-03T17:28:53.114Z 22 440-459 129 129 129 236 236 236 F F F
139552024-02-03T17:28:53.114Z 23 460-479 128 128 128 235 235 235 F F F
139562024-02-03T17:28:53.114Z 24 480-499 124 124 124 225 225 225 F F F
139572024-02-03T17:28:53.114Z 25 500-519 129 129 129 236 236 236 F F F
139582024-02-03T17:28:53.114Z 26 520-539 124 124 124 225 225 225 F F F
139592024-02-03T17:28:53.114Z 27 540-559 128 128 128 234 234 234 F F F
139602024-02-03T17:28:53.114Z 28 560-579 122 122 122 223 223 223 F F F
139612024-02-03T17:28:53.114Z 29 580-599 124 124 124 225 225 225 F F F
139622024-02-03T17:28:53.114ZMax gen: 129, Max flush: 236
139632024-02-03T17:28:53.114Z
139642024-02-03T17:28:53.115ZBegin loop 65
139652024-02-03T17:28:53.115ZDownstairs to restart: 1
139662024-02-03T17:28:53.115Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.291062113Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655}
139672024-02-03T17:28:53.115Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.291548761Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655}
139682024-02-03T17:28:53.115Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.291687342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655}
139692024-02-03T17:28:53.115Z{"msg":"Crucible c5fdf37c-250b-41ff-81ea-b46fc540dafa has session id: 4a3705a1-4d94-4a53-a9ac-a4e173f3e229","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.291728121Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139702024-02-03T17:28:53.115Z{Crucible runtime is spawned
139712024-02-03T17:28:53.115Z"msg":"Upstairs opts: Upstairs UUID: c5fdf37c-250b-41ff-81ea-b46fc540dafa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.291854021Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139722024-02-03T17:28:53.115Z{"msg":"Crucible stats registered with UUID: c5fdf37c-250b-41ff-81ea-b46fc540dafa","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.291900401Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139732024-02-03T17:28:53.115Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.291958616Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655}
139742024-02-03T17:28:53.115Z{"msg":"c5fdf37c-250b-41ff-81ea-b46fc540dafa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292137467Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139752024-02-03T17:28:53.116Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292202393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139762024-02-03T17:28:53.116Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292250342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139772024-02-03T17:28:53.116Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.29227521Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139782024-02-03T17:28:53.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292322934Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139792024-02-03T17:28:53.163Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292395303Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139802024-02-03T17:28:53.163Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292444252Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139812024-02-03T17:28:53.163Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292497097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139822024-02-03T17:28:53.163Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292560543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139832024-02-03T17:28:53.163Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292625167Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139842024-02-03T17:28:53.164Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292724035Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139852024-02-03T17:28:53.164Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.29281206Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139862024-02-03T17:28:53.164Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.292932492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139872024-02-03T17:28:53.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.294064408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139882024-02-03T17:28:53.164Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.294166376Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139892024-02-03T17:28:53.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.294195014Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139902024-02-03T17:28:53.164Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.294230818Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139912024-02-03T17:28:53.164Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.295013164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139922024-02-03T17:28:53.164Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.295226431Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139932024-02-03T17:28:53.164Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.295659208Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139942024-02-03T17:28:53.164Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.295844324Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139952024-02-03T17:28:53.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.295909178Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139962024-02-03T17:28:53.164Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.295980187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139972024-02-03T17:28:53.164Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.296044441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139982024-02-03T17:28:53.164Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.296078265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
139992024-02-03T17:28:53.165Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.296156728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140002024-02-03T17:28:53.165Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.29621333Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140012024-02-03T17:28:53.165Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.296499623Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140022024-02-03T17:28:53.165Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297193415Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140032024-02-03T17:28:53.165Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297261373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140042024-02-03T17:28:53.165Z{"msg":"[0]R flush_numbers[0..12]: [231, 235, 211, 224, 234, 235, 235, 235, 225, 224, 231, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.29730427Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140052024-02-03T17:28:53.165Z{"msg":"[0]R generation[0..12]: [127, 128, 116, 123, 128, 128, 128, 128, 124, 123, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297351853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140062024-02-03T17:28:53.165Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297396705Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140072024-02-03T17:28:53.165Z{"msg":"[1]R flush_numbers[0..12]: [231, 235, 211, 224, 234, 235, 235, 235, 225, 224, 231, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.29744067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140082024-02-03T17:28:53.165Z{"msg":"[1]R generation[0..12]: [127, 128, 116, 123, 128, 128, 128, 128, 124, 123, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297466715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140092024-02-03T17:28:53.165Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297508655Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"Wait for a query_work_queue command to finish before sending IO
140102024-02-03T17:28:53.165Z}
140112024-02-03T17:28:53.165Z{"msg":"[2]R flush_numbers[0..12]: [231, 235, 211, 224, 234, 235, 235, 235, 225, 224, 231, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297628082Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140122024-02-03T17:28:53.165Z{"msg":"[2]R generation[0..12]: [127, 128, 116, 123, 128, 128, 128, 128, 124, 123, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297684814Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140132024-02-03T17:28:53.165Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297730015Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140142024-02-03T17:28:53.165Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297776451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140152024-02-03T17:28:53.165Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297826679Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140162024-02-03T17:28:53.165Z{"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297871954Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140172024-02-03T17:28:53.165Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297917711Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140182024-02-03T17:28:53.165Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.297970282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140192024-02-03T17:28:53.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.298018338Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140202024-02-03T17:28:53.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.298042278Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140212024-02-03T17:28:53.166Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.298085698Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140222024-02-03T17:28:53.166Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.298127737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140232024-02-03T17:28:53.166Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.298172519Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140242024-02-03T17:28:53.166Z{"msg":"c5fdf37c-250b-41ff-81ea-b46fc540dafa is now active with session: 4a3705a1-4d94-4a53-a9ac-a4e173f3e229","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.298195341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140252024-02-03T17:28:53.166Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:46.298233924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655,"session_id":"4a3705a1-4d94-4a53-a9ac-a4e173f3e229"}
140262024-02-03T17:28:53.166Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140272024-02-03T17:28:53.166Z,"time":"2024-02-03T17:28:46.298312886Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655}
140282024-02-03T17:28:53.166ZLoading write count information from file "/tmp/test_repair_verify.data"
140292024-02-03T17:28:53.166ZRun Repair workload
140302024-02-03T17:28:53.166ZError: Difference in extent metadata found!
140312024-02-03T17:28:53.166Z01/30 Flush
140322024-02-03T17:28:53.166Z02/30 Write block 420 len 16384 data: 15 13 12 12
140332024-02-03T17:28:53.166Z03/30 Write block 405 len 16384 data: 13 12 13 10
140342024-02-03T17:28:53.166Z04/30 Read block 55 len 40960
140352024-02-03T17:28:53.166Z05/30 Flush
140362024-02-03T17:28:53.166Z06/30 Write block 96 len 40960 data: 9 10 7 9 10 10 10 11 9 12
140372024-02-03T17:28:53.166Z07/30 Write block 61 len 4096 data: 11
140382024-02-03T17:28:53.166Z08/30 Read block 112 len 4096
140392024-02-03T17:28:53.166Z09/30 Write block 385 len 24576 data: 7 6 7 7 8 8
140402024-02-03T17:28:53.166Z10/30 Read block 38 len 4096
140412024-02-03T17:28:53.166Z11/30 Write block 438 len 4096 data: 12
140422024-02-03T17:28:53.167Z12/30 Write block 514 len 16384 data: 5 8 6 8
140432024-02-03T17:28:53.167Z13/30 Read block 544 len 8192
140442024-02-03T17:28:53.167Z14/30 Write block 331 len 28672 data: 9 9 10 10 9 9 8
140452024-02-03T17:28:53.167Z15/30 Read block 251 len 12288
140462024-02-03T17:28:53.167Z16/30 Read block 101 len 16384
140472024-02-03T17:28:53.167Z17/30 Write block 586 len 28672 data: 12 9 9 12 13 13 11
140482024-02-03T17:28:53.167Z18/30 Read block 33 len 40960
140492024-02-03T17:28:53.167Z19/30 Write block 215 len 16384 data: 9 7 8 8
140502024-02-03T17:28:53.167Z20/30 Read block 95 len 12288
140512024-02-03T17:28:53.167Z21/30 Write block 7 len 36864 data: 12 11 11 15 13 13 11 9 7
140522024-02-03T17:28:53.167Z22/30 Write block 260 len 16384 data: 8 10 12 11
140532024-02-03T17:28:53.167Z23/30 Read block 414 len 32768
140542024-02-03T17:28:53.167Z24/30 Write block 453 len 40960 data: 7 5 5 9 12 12 13 10 9 11
140552024-02-03T17:28:53.167Z25/30 Read block 65 len 12288
140562024-02-03T17:28:53.167Z26/30 Write block 8 len 20480 data: 12 12 16 14 14
140572024-02-03T17:28:53.167Z27/30 Read block 459 len 16384
140582024-02-03T17:28:53.167Z28/30 Read block 31 len 16384
140592024-02-03T17:28:53.167Z29/30 Read block 251 len 40960
140602024-02-03T17:28:53.167Z30/30 Read block 96 len 32768
140612024-02-03T17:28:53.167Z----------------------------------------------------------------
140622024-02-03T17:28:53.167Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:25
140632024-02-03T17:28:53.167ZStates: Active Active Active
140642024-02-03T17:28:53.167ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140652024-02-03T17:28:53.167Z 6 Acked 1005 Write 10 Done Done Done false
140662024-02-03T17:28:53.168Z 7 Acked 1006 Write 1 Done Done Done false
140672024-02-03T17:28:53.168Z 8 Acked 1007 Read 1 Done Done Done false
140682024-02-03T17:28:53.168Z 9 Acked 1008 Write 6 Done Sent Done false
140692024-02-03T17:28:53.168Z 10 Acked 1009 Read 1 Done Sent Done false
140702024-02-03T17:28:53.168Z 11 Acked 1010 Write 1 Done Sent Done false
140712024-02-03T17:28:53.168Z 12 Acked 1011 Write 4 Done Sent Done false
140722024-02-03T17:28:53.168Z 13 Acked 1012 Read 2 Done Sent Done false
140732024-02-03T17:28:53.168Z 14 Acked 1013 Write 7 Done Sent Done false
140742024-02-03T17:28:53.168Z 15 Acked 1014 Read 3 Done Sent Done false
140752024-02-03T17:28:53.168Z 16 Acked 1015 Read 4 Done Sent Done false
140762024-02-03T17:28:53.168Z 17 Acked 1016 Write 7 Done Sent Done false
140772024-02-03T17:28:53.168Z 18 Acked 1017 Read 10 Done Sent Done false
140782024-02-03T17:28:53.185Z 19 Acked 1018 Write 4 Done Sent Done false
140792024-02-03T17:28:53.185Z 20 Acked 1019 Read 3 Done Sent Done false
140802024-02-03T17:28:53.185Z 21 Acked 1020 Write 9 Done Sent Done false
140812024-02-03T17:28:53.185Z 22 Acked 1021 Write 4 Done Sent Done false
140822024-02-03T17:28:53.185Z 23 Acked 1022 Read 8 Done Sent Done false
140832024-02-03T17:28:53.185Z 24 Acked 1023 Write 10 Done Sent Done false
140842024-02-03T17:28:53.185Z 25 Acked 1024 Read 3 Done Sent Done false
140852024-02-03T17:28:53.185Z 26 Acked 1025 Write 5 Done Sent Done false
140862024-02-03T17:28:53.185Z 27 Acked 1026 Read 4 Done Sent Done false
140872024-02-03T17:28:53.185Z 28 Acked 1027 Read 4 Done Sent Done false
140882024-02-03T17:28:53.186Z 29 Acked 1028 Read 10 Done Sent Done false
140892024-02-03T17:28:53.186Z 30 Acked 1029 Read 8 Done Sent Done false
140902024-02-03T17:28:53.186Z STATES DS:0 DS:1 DS:2 TOTAL
140912024-02-03T17:28:53.186Z New 0 0 0 0
140922024-02-03T17:28:53.186Z Sent 0 22 0 22
140932024-02-03T17:28:53.186Z Done 25 3 25 53
140942024-02-03T17:28:53.186Z Skipped 0 0 0 0
140952024-02-03T17:28:53.186Z Error 0 0 0 0
140962024-02-03T17:28:53.186ZLast Flush: 1004 1004 1004
140972024-02-03T17:28:53.186ZDownstairs last five completed: 1004 1003 1002 1001 1000
140982024-02-03T17:28:53.186ZUpstairs last five completed: 30 29 28 27 26
140992024-02-03T17:28:53.186Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:46.341665827Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1655}
141002024-02-03T17:28:53.187ZWrote out file "/tmp/test_repair_verify.data"
141012024-02-03T17:28:53.187Z
141022024-02-03T17:28:53.187ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141032024-02-03T17:28:53.187ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141042024-02-03T17:28:53.187Z 0 000-019 127 127 127 231 231 231 T F T <---
141052024-02-03T17:28:53.187Z 1 020-039 128 128 128 235 235 235 F F F
141062024-02-03T17:28:53.187Z 2 040-059 116 116 116 211 211 211 F F F
141072024-02-03T17:28:53.187Z 3 060-079 123 123 123 224 224 224 T T T
141082024-02-03T17:28:53.187Z 4 080-099 128 128 128 234 234 234 T T T
141092024-02-03T17:28:53.187Z 5 100-119 128 128 128 235 235 235 T T T
141102024-02-03T17:28:53.187Z 6 120-139 128 128 128 235 235 235 F F F
141112024-02-03T17:28:53.187Z 7 140-159 128 128 128 235 235 235 F F F
141122024-02-03T17:28:53.187Z 8 160-179 124 124 124 225 225 225 F F F
141132024-02-03T17:28:53.188Z 9 180-199 123 123 123 224 224 224 F F F
141142024-02-03T17:28:53.188Z 10 200-219 127 127 127 231 231 231 T F T <---
141152024-02-03T17:28:53.188Z 11 220-239 120 120 120 220 220 220 F F F
141162024-02-03T17:28:53.188Z 12 240-259 129 129 129 236 236 236 F F F
141172024-02-03T17:28:53.188Z 13 260-279 129 129 129 236 236 236 T F T <---
141182024-02-03T17:28:53.188Z 14 280-299 127 127 127 231 231 231 F F F
141192024-02-03T17:28:53.188Z 15 300-319 124 124 124 225 225 225 F F F
141202024-02-03T17:28:53.188Z 16 320-339 127 127 127 231 231 231 T F T <---
141212024-02-03T17:28:53.189Z 17 340-359 124 124 124 225 225 225 F F F
141222024-02-03T17:28:53.189Z 18 360-379 123 123 123 224 224 224 F F F
141232024-02-03T17:28:53.189Z 19 380-399 116 116 116 208 208 208 T F T <---
141242024-02-03T17:28:53.189Z 20 400-419 130 130 130 238 238 238 F F F
141252024-02-03T17:28:53.189Z 21 420-439 130 130 130 238 238 238 T F T <---
141262024-02-03T17:28:53.189Z 22 440-459 129 129 129 236 236 236 T F T <---
141272024-02-03T17:28:53.189Z 23 460-479 128 128 128 235 235 235 T F T <---
141282024-02-03T17:28:53.189Z 24 480-499 124 124 124 225 225 225 F F F
141292024-02-03T17:28:53.189Z 25 500-519 129 129 129 236 236 236 T F T <---
141302024-02-03T17:28:53.189Z 26 520-539 124 124 124 225 225 225 F F F
141312024-02-03T17:28:53.189Z 27 540-559 128 128 128 234 234 234 F F F
141322024-02-03T17:28:53.189Z 28 560-579 122 122 122 223 223 223 F F F
141332024-02-03T17:28:53.190Z 29 580-599 124 124 124 225 225 225 T F T <---
141342024-02-03T17:28:53.190ZMax gen: 130, Max flush: 238
141352024-02-03T17:28:53.190ZA Difference in extent metadata is expected here
141362024-02-03T17:28:53.190ZOn loop 65
141372024-02-03T17:28:53.190Z
141382024-02-03T17:28:53.190ZVerifying data now
141392024-02-03T17:28:53.386ZNo change after verify
141402024-02-03T17:28:53.386ZLoop: 65 Downstairs dump after verify (and repair):
141412024-02-03T17:28:53.408ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141422024-02-03T17:28:53.409Z 0 000-019 131 131 131 239 239 239 F F F
141432024-02-03T17:28:53.416Z 1 020-039 128 128 128 235 235 235 F F F
141442024-02-03T17:28:53.416Z 2 040-059 116 116 116 211 211 211 F F F
141452024-02-03T17:28:53.416Z 3 060-079 131 131 131 239 239 239 F F F
141462024-02-03T17:28:53.416Z 4 080-099 131 131 131 239 239 239 F F F
141472024-02-03T17:28:53.416Z 5 100-119 131 131 131 239 239 239 F F F
141482024-02-03T17:28:53.416Z 6 120-139 128 128 128 235 235 235 F F F
141492024-02-03T17:28:53.416Z 7 140-159 128 128 128 235 235 235 F F F
141502024-02-03T17:28:53.416Z 8 160-179 124 124 124 225 225 225 F F F
141512024-02-03T17:28:53.416Z 9 180-199 123 123 123 224 224 224 F F F
141522024-02-03T17:28:53.417Z 10 200-219 131 131 131 239 239 239 F F F
141532024-02-03T17:28:53.417Z 11 220-239 120 120 120 220 220 220 F F F
141542024-02-03T17:28:53.417Z 12 240-259 129 129 129 236 236 236 F F F
141552024-02-03T17:28:53.417Z 13 260-279 131 131 131 239 239 239 F F F
141562024-02-03T17:28:53.417Z 14 280-299 127 127 127 231 231 231 F F F
141572024-02-03T17:28:53.417Z 15 300-319 124 124 124 225 225 225 F F F
141582024-02-03T17:28:53.417Z 16 320-339 131 131 131 239 239 239 F F F
141592024-02-03T17:28:53.417Z 17 340-359 124 124 124 225 225 225 F F F
141602024-02-03T17:28:53.417Z 18 360-379 123 123 123 224 224 224 F F F
141612024-02-03T17:28:53.417Z 19 380-399 131 131 131 239 239 239 F F F
141622024-02-03T17:28:53.417Z 20 400-419 130 130 130 238 238 238 F F F
141632024-02-03T17:28:53.417Z 21 420-439 131 131 131 239 239 239 F F F
141642024-02-03T17:28:53.418Z 22 440-459 131 131 131 239 239 239 F F F
141652024-02-03T17:28:53.418Z 23 460-479 131 131 131 239 239 239 F F F
141662024-02-03T17:28:53.418Z 24 480-499 124 124 124 225 225 225 F F F
141672024-02-03T17:28:53.418Z 25 500-519 131 131 131 239 239 239 F F F
141682024-02-03T17:28:53.418Z 26 520-539 124 124 124 225 225 225 F F F
141692024-02-03T17:28:53.418Z 27 540-559 128 128 128 234 234 234 F F F
141702024-02-03T17:28:53.418Z 28 560-579 122 122 122 223 223 223 F F F
141712024-02-03T17:28:53.418Z 29 580-599 131 131 131 239 239 239 F F F
141722024-02-03T17:28:53.418ZMax gen: 131, Max flush: 239
141732024-02-03T17:28:53.418Z
141742024-02-03T17:28:53.419ZBegin loop 66
141752024-02-03T17:28:53.419ZDownstairs to restart: 1
141762024-02-03T17:28:53.446Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.61702242Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663}
141772024-02-03T17:28:53.446Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.617510296Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663}
141782024-02-03T17:28:53.455Z{Crucible runtime is spawned
141792024-02-03T17:28:53.455Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.617696534Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663}
141802024-02-03T17:28:53.455Z{"msg":"Crucible 11d4e791-5a65-4146-8026-0519667696ca has session id: aad92102-a541-4537-b8bb-859558120585","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.617782662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
141812024-02-03T17:28:53.455Z{"msg":"Upstairs opts: Upstairs UUID: 11d4e791-5a65-4146-8026-0519667696ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.617826887Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
141822024-02-03T17:28:53.456Z{"msg":"Crucible stats registered with UUID: 11d4e791-5a65-4146-8026-0519667696ca","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.617851169Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
141832024-02-03T17:28:53.456Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.61787238Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663}
141842024-02-03T17:28:53.456Z{"msg":"11d4e791-5a65-4146-8026-0519667696ca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618041131Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
141852024-02-03T17:28:53.456Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618093095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141862024-02-03T17:28:53.456Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618119541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141872024-02-03T17:28:53.456Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618141877Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141882024-02-03T17:28:53.456Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618176757Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141892024-02-03T17:28:53.456Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618255151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141902024-02-03T17:28:53.456Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618300926Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141912024-02-03T17:28:53.456Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618378236Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141922024-02-03T17:28:53.457Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618422192Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141932024-02-03T17:28:53.457Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618490824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141942024-02-03T17:28:53.457Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618709354Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141952024-02-03T17:28:53.457Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618811089Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141962024-02-03T17:28:53.457Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.618844791Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141972024-02-03T17:28:53.457Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.620205546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141982024-02-03T17:28:53.457Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.620273998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
141992024-02-03T17:28:53.457Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.620319362Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142002024-02-03T17:28:53.457Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.620433374Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142012024-02-03T17:28:53.457Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.621227148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142022024-02-03T17:28:53.457Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.621289557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142032024-02-03T17:28:53.457Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.621317531Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142042024-02-03T17:28:53.457Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.621465144Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142052024-02-03T17:28:53.458Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.622402187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142062024-02-03T17:28:53.458Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.622477599Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142072024-02-03T17:28:53.458Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.622522128Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
142082024-02-03T17:28:53.458Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.622575669Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142092024-02-03T17:28:53.458Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.622601785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142102024-02-03T17:28:53.458Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.62265058Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
142112024-02-03T17:28:53.458Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.622674692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142122024-02-03T17:28:53.458Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623352019Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142132024-02-03T17:28:53.458Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623411709Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142142024-02-03T17:28:53.458Z{"msg":"[0]R flush_numbers[0..12]: [239, 235, 211, 239, 239, 239, 235, 235, 225, 224, 239, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623439451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142152024-02-03T17:28:53.458Z{"msg":"[0]R generation[0..12]: [131, 128, 116, 131, 131, 131, 128, 128, 124, 123, 131, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623482945Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142162024-02-03T17:28:53.458Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623509028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142172024-02-03T17:28:53.458Z{"msg":"[1]R flush_numbers[0..12]: [239, 235, 211, 239, 239, 239, 235, 235, 225, 224, 239, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623551882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142182024-02-03T17:28:53.459Z{"msg":"[1]R generation[0..12]: [131, 128, 116, 131, 131, 131, 128, 128, 124, 123, 131, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623577517Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142192024-02-03T17:28:53.459Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623624373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142202024-02-03T17:28:53.459Z{"msg":"[2]R flush_numbers[0..12]: [239, 235, 211, 239, 239, 239, 235, 235, 225, 224, 239, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623649944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142212024-02-03T17:28:53.459Z{"msg":"[2]R generation[0..12]: [131, 128, 116, 131, 131, 131, 128, 128, 124, 123, 131, 120]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
142222024-02-03T17:28:53.459Z,"time":"2024-02-03T17:28:52.623709861Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142232024-02-03T17:28:53.459Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623756582Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142242024-02-03T17:28:53.459Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623780459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142252024-02-03T17:28:53.459Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623834712Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142262024-02-03T17:28:53.459Z{"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623860018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142272024-02-03T17:28:53.459Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623920444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142282024-02-03T17:28:53.459Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623946153Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
142292024-02-03T17:28:53.459Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.623966731Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142302024-02-03T17:28:53.459Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.624010509Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142312024-02-03T17:28:53.459Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.624034113Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"aad92102-a541-4537-b8bb-859558120585"}
142322024-02-03T17:28:53.459Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.624125754Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
142332024-02-03T17:28:53.459Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.624158577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
142342024-02-03T17:28:53.459Z{"msg":"11d4e791-5a65-4146-8026-0519667696ca is now active with session: aad92102-a541-4537-b8bb-859558120585","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.624181557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
142352024-02-03T17:28:53.459Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.624221153Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663,"session_id":"aad92102-a541-4537-b8bb-859558120585"}
142362024-02-03T17:28:53.459Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:52.624243851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663}
142372024-02-03T17:28:53.460ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142382024-02-03T17:28:53.460ZLoading write count information from file "/tmp/test_repair_verify.data"
142392024-02-03T17:28:53.460ZRun Repair workload
142402024-02-03T17:28:53.460Z01/30 Write block 580 len 12288 data: 14 14 13
142412024-02-03T17:28:53.468Z02/30 Write block 132 len 4096 data: 16
142422024-02-03T17:28:53.469Z03/30 Write block 453 len 32768 data: 8 6 6 10 13 13 14 11
142432024-02-03T17:28:53.469Z04/30 Read block 218 len 28672
142442024-02-03T17:28:53.469Z05/30 Read block 249 len 4096
142452024-02-03T17:28:53.469Z06/30 Write block 544 len 8192 data: 12 13
142462024-02-03T17:28:53.469Z07/30 Write block 311 len 20480 data: 9 6 8 8 8
142472024-02-03T17:28:53.469Z08/30 Write block 237 len 8192 data: 12 9
142482024-02-03T17:28:53.469Z09/30 Flush
142492024-02-03T17:28:53.485Z10/30 Write block 343 len 28672 data: 7 8 8 7 9 11 10
142502024-02-03T17:28:53.485Z11/30 Read block 236 len 24576
142512024-02-03T17:28:53.486Z12/30 Flush
142522024-02-03T17:28:53.486Z13/30 Write block 19 len 4096 data: 6
142532024-02-03T17:28:53.486Z14/30 Flush
142542024-02-03T17:28:53.486Z15/30 Write block 179 len 40960 data: 9 10 11 10 9 7 8 8 10 9
142552024-02-03T17:28:53.486Z16/30 Write block 426 len 12288 data: 7 7 7
142562024-02-03T17:28:53.486Z17/30 Read block 62 len 28672
142572024-02-03T17:28:53.486Z18/30 Read block 365 len 32768
142582024-02-03T17:28:53.486Z19/30 Flush
142592024-02-03T17:28:53.495Z20/30 Write block 584 len 24576 data: 11 9 13 10 10 13
142602024-02-03T17:28:53.495Z21/30 Read block 490 len 8192
142612024-02-03T17:28:53.495Z22/30 Flush
142622024-02-03T17:28:53.495Z23/30 Flush
142632024-02-03T17:28:53.495Z24/30 Write block 259 len 8192 data: 6 9
142642024-02-03T17:28:53.495Z25/30 Read block 188 len 24576
142652024-02-03T17:28:53.496Z26/30 Read block 544 len 28672
142662024-02-03T17:28:53.517Z27/30 Read block 99 len 4096
142672024-02-03T17:28:53.517Z28/30 Write block 595 len 12288 data: 8 8 8
142682024-02-03T17:28:53.517Z29/30 Read block 421 len 4096
142692024-02-03T17:28:53.517Z30/30 Read block 566 len 40960
142702024-02-03T17:28:53.517Z----------------------------------------------------------------
142712024-02-03T17:28:53.517Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142722024-02-03T17:28:53.517ZStates: Active Active Active
142732024-02-03T17:28:53.517ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142742024-02-03T17:28:53.517Z 1 Acked 1000 Write 3 Done Done Done false
142752024-02-03T17:28:53.517Z 2 Acked 1001 Write 1 Done Done Done false
142762024-02-03T17:28:53.517Z 3 Acked 1002 Write 8 Done Sent Done false
142772024-02-03T17:28:53.517Z 4 Acked 1003 Read 7 Done Sent Done false
142782024-02-03T17:28:53.517Z 5 Acked 1004 Read 1 Done Sent Done false
142792024-02-03T17:28:53.517Z 6 Acked 1005 Write 2 Done Sent Done false
142802024-02-03T17:28:53.518Z 7 Acked 1006 Write 5 Done Sent Done false
142812024-02-03T17:28:53.518Z 8 Acked 1007 Write 2 Done Sent Done false
142822024-02-03T17:28:53.518Z 9 Acked 1008 Flush 0 Done Sent Done false
142832024-02-03T17:28:53.518Z 10 Acked 1009 Write 7 Done Sent Done false
142842024-02-03T17:28:53.518Z 11 Acked 1010 Read 6 Done Sent Done false
142852024-02-03T17:28:53.518Z 12 Acked 1011 Flush 0 Done Sent Done false
142862024-02-03T17:28:53.518Z 13 Acked 1012 Write 1 Done Sent Done false
142872024-02-03T17:28:53.518Z 14 Acked 1013 Flush 0 Done Sent Done false
142882024-02-03T17:28:53.518Z 15 Acked 1014 Write 10 Done Sent Done false
142892024-02-03T17:28:53.518Z 16 Acked 1015 Write 3 Done Sent Done false
142902024-02-03T17:28:53.518Z 17 Acked 1016 Read 7 Done Sent Done false
142912024-02-03T17:28:53.518Z 18 Acked 1017 Read 8 Done Sent Done false
142922024-02-03T17:28:53.518Z 19 Acked 1018 Flush 0 Done Sent Done false
142932024-02-03T17:28:53.518Z 20 Acked 1019 Write 6 Done Sent Done false
142942024-02-03T17:28:53.518Z 21 Acked 1020 Read 2 Done Sent Done false
142952024-02-03T17:28:53.518Z 22 Acked 1021 Flush 0 Done Sent Done false
142962024-02-03T17:28:53.518Z 23 Acked 1022 Flush 0 Done Sent Done false
142972024-02-03T17:28:53.518Z 24 Acked 1023 Write 2 Done Sent Done false
142982024-02-03T17:28:53.518Z 25 Acked 1024 Read 6 Done Sent Done false
142992024-02-03T17:28:53.518Z 26 Acked 1025 Read 7 Done Sent Done false
143002024-02-03T17:28:53.518Z 27 Acked 1026 Read 1 Done Sent Done false
143012024-02-03T17:28:53.518Z 28 Acked 1027 Write 3 Done Sent Done false
143022024-02-03T17:28:53.519Z 29 Acked 1028 Read 1 Done Sent Done false
143032024-02-03T17:28:53.519Z 30 Acked 1029 Read 10 Done Sent Done false
143042024-02-03T17:28:53.519Z STATES DS:0 DS:1 DS:2 TOTAL
143052024-02-03T17:28:53.519Z New 0 0 0 0
143062024-02-03T17:28:53.519Z Sent 0 28 0 28
143072024-02-03T17:28:53.519Z Done 30 2 30 62
143082024-02-03T17:28:53.519Z Skipped 0 0 0 0
143092024-02-03T17:28:53.519Z Error 0 0 0 0
143102024-02-03T17:28:53.519ZLast Flush: 1022 0 1022
143112024-02-03T17:28:53.519ZDownstairs last five completed:
143122024-02-03T17:28:53.519ZUpstairs last five completed: 30 29 28 27 26
143132024-02-03T17:28:53.520Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:52.672343309Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1663}
143142024-02-03T17:28:53.520ZWrote out file "/tmp/test_repair_verify.data"
143152024-02-03T17:28:53.520Z
143162024-02-03T17:28:53.520ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143172024-02-03T17:28:53.542ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143182024-02-03T17:28:53.542Z 0 000-019 132 131 132 242 239 242 F F F <---
143192024-02-03T17:28:53.553Z 1 020-039 128 128 128 235 235 235 F F F
143202024-02-03T17:28:53.553Z 2 040-059 116 116 116 211 211 211 F F F
143212024-02-03T17:28:53.553Z 3 060-079 131 131 131 239 239 239 F F F
143222024-02-03T17:28:53.554Z 4 080-099 131 131 131 239 239 239 F F F
143232024-02-03T17:28:53.554Z 5 100-119 131 131 131 239 239 239 F F F
143242024-02-03T17:28:53.554Z 6 120-139 132 128 132 240 235 240 F T F <---
143252024-02-03T17:28:53.554Z 7 140-159 128 128 128 235 235 235 F F F
143262024-02-03T17:28:53.554Z 8 160-179 132 124 132 243 225 243 F F F <---
143272024-02-03T17:28:53.554Z 9 180-199 132 123 132 243 224 243 F F F <---
143282024-02-03T17:28:53.554Z 10 200-219 131 131 131 239 239 239 F F F
143292024-02-03T17:28:53.554Z 11 220-239 132 120 132 240 220 240 F F F <---
143302024-02-03T17:28:53.554Z 12 240-259 129 129 129 236 236 236 T F T <---
143312024-02-03T17:28:53.554Z 13 260-279 131 131 131 239 239 239 T F T <---
143322024-02-03T17:28:53.554Z 14 280-299 127 127 127 231 231 231 F F F
143332024-02-03T17:28:53.554Z 15 300-319 132 124 132 240 225 240 F F F <---
143342024-02-03T17:28:53.555Z 16 320-339 131 131 131 239 239 239 F F F
143352024-02-03T17:28:53.555Z 17 340-359 132 124 132 241 225 241 F F F <---
143362024-02-03T17:28:53.555Z 18 360-379 123 123 123 224 224 224 F F F
143372024-02-03T17:28:53.555Z 19 380-399 131 131 131 239 239 239 F F F
143382024-02-03T17:28:53.555Z 20 400-419 130 130 130 238 238 238 F F F
143392024-02-03T17:28:53.555Z 21 420-439 132 131 132 243 239 243 F F F <---
143402024-02-03T17:28:53.555Z 22 440-459 132 131 132 240 239 240 F F F <---
143412024-02-03T17:28:53.555Z 23 460-479 132 131 132 240 239 240 F F F <---
143422024-02-03T17:28:53.555Z 24 480-499 124 124 124 225 225 225 F F F
143432024-02-03T17:28:53.555Z 25 500-519 131 131 131 239 239 239 F F F
143442024-02-03T17:28:53.555Z 26 520-539 124 124 124 225 225 225 F F F
143452024-02-03T17:28:53.556Z 27 540-559 132 128 132 240 234 240 F F F <---
143462024-02-03T17:28:53.556Z 28 560-579 122 122 122 223 223 223 F F F
143472024-02-03T17:28:53.556Z 29 580-599 132 131 132 244 239 244 T T T <---
143482024-02-03T17:28:53.556ZMax gen: 132, Max flush: 244
143492024-02-03T17:28:53.556ZError: Difference in extent metadata found!
143502024-02-03T17:28:53.556ZA Difference in extent metadata is expected here
143512024-02-03T17:28:53.556ZOn loop 66
143522024-02-03T17:28:53.556Z
143532024-02-03T17:28:53.556ZVerifying data now
143542024-02-03T17:28:58.761ZNo change after verify
143552024-02-03T17:28:58.761ZLoop: 66 Downstairs dump after verify (and repair):
143562024-02-03T17:28:58.788ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143572024-02-03T17:28:58.788Z 0 000-019 132 132 132 242 242 242 F F F
143582024-02-03T17:28:58.788Z 1 020-039 128 128 128 235 235 235 F F F
143592024-02-03T17:28:58.788Z 2 040-059 116 116 116 211 211 211 F F F
143602024-02-03T17:28:58.791Z 3 060-079 131 131 131 239 239 239 F F F
143612024-02-03T17:28:58.791Z 4 080-099 131 131 131 239 239 239 F F F
143622024-02-03T17:28:58.791Z 5 100-119 131 131 131 239 239 239 F F F
143632024-02-03T17:28:58.791Z 6 120-139 132 132 132 240 240 240 F F F
143642024-02-03T17:28:58.792Z 7 140-159 128 128 128 235 235 235 F F F
143652024-02-03T17:28:58.792Z 8 160-179 132 132 132 243 243 243 F F F
143662024-02-03T17:28:58.792Z 9 180-199 132 132 132 243 243 243 F F F
143672024-02-03T17:28:58.792Z 10 200-219 131 131 131 239 239 239 F F F
143682024-02-03T17:28:58.792Z 11 220-239 132 132 132 240 240 240 F F F
143692024-02-03T17:28:58.792Z 12 240-259 133 133 133 245 245 245 F F F
143702024-02-03T17:28:58.792Z 13 260-279 133 133 133 245 245 245 F F F
143712024-02-03T17:28:58.792Z 14 280-299 127 127 127 231 231 231 F F F
143722024-02-03T17:28:58.792Z 15 300-319 132 132 132 240 240 240 F F F
143732024-02-03T17:28:58.792Z 16 320-339 131 131 131 239 239 239 F F F
143742024-02-03T17:28:58.792Z 17 340-359 132 132 132 241 241 241 F F F
143752024-02-03T17:28:58.792Z 18 360-379 123 123 123 224 224 224 F F F
143762024-02-03T17:28:58.792Z 19 380-399 131 131 131 239 239 239 F F F
143772024-02-03T17:28:58.792Z 20 400-419 130 130 130 238 238 238 F F F
143782024-02-03T17:28:58.792Z 21 420-439 132 132 132 243 243 243 F F F
143792024-02-03T17:28:58.792Z 22 440-459 132 132 132 240 240 240 F F F
143802024-02-03T17:28:58.792Z 23 460-479 132 132 132 240 240 240 F F F
143812024-02-03T17:28:58.792Z 24 480-499 124 124 124 225 225 225 F F F
143822024-02-03T17:28:58.792Z 25 500-519 131 131 131 239 239 239 F F F
143832024-02-03T17:28:58.793Z 26 520-539 124 124 124 225 225 225 F F F
143842024-02-03T17:28:58.793Z 27 540-559 132 132 132 240 240 240 F F F
143852024-02-03T17:28:58.793Z 28 560-579 122 122 122 223 223 223 F F F
143862024-02-03T17:28:58.793Z 29 580-599 133 133 133 245 245 245 F F F
143872024-02-03T17:28:58.793ZMax gen: 133, Max flush: 245
143882024-02-03T17:28:58.793Z
143892024-02-03T17:28:58.793ZBegin loop 67
143902024-02-03T17:28:58.793ZDownstairs to restart: 0
143912024-02-03T17:28:58.823Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.994540279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671}
143922024-02-03T17:28:58.827Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995041805Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671}
143932024-02-03T17:28:58.827Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995118926Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671}
143942024-02-03T17:28:58.827Z{"msg":"Crucible 1d3025fa-2f01-47fb-94e9-12952ab769d7 has session id: 5025e229-b8b3-4f95-81f2-e14907ef0d58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995152709Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
143952024-02-03T17:28:58.827Z{"msg":"Upstairs opts: Upstairs UUID: 1d3025fa-2f01-47fb-94e9-12952ab769d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
143962024-02-03T17:28:58.827Z,"time":"2024-02-03T17:28:57.995311703Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
143972024-02-03T17:28:58.827Z{"msg":"Crucible stats registered with UUID: 1d3025fa-2f01-47fb-94e9-12952ab769d7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995485028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
143982024-02-03T17:28:58.827Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.99556025Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671}
143992024-02-03T17:28:58.827Z{"msg":"1d3025fa-2f01-47fb-94e9-12952ab769d7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995635399Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144002024-02-03T17:28:58.827Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995676582Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144012024-02-03T17:28:58.827Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995756293Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144022024-02-03T17:28:58.827Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995790171Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144032024-02-03T17:28:58.827Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995819968Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144042024-02-03T17:28:58.827Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.99587906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144052024-02-03T17:28:58.827Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.99590647Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144062024-02-03T17:28:58.827Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995928424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144072024-02-03T17:28:58.827Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.995983259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144082024-02-03T17:28:58.827Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.996022243Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144092024-02-03T17:28:58.828Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.996193723Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144102024-02-03T17:28:58.828Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.996442459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144112024-02-03T17:28:58.828Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.996493523Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144122024-02-03T17:28:58.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.997791714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144132024-02-03T17:28:58.828Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.997851378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144142024-02-03T17:28:58.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.997875675Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144152024-02-03T17:28:58.828Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.997951137Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144162024-02-03T17:28:58.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.997990977Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144172024-02-03T17:28:58.828Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.998151617Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144182024-02-03T17:28:58.832Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.998788097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144192024-02-03T17:28:58.832Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.999181277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144202024-02-03T17:28:58.832Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.999455083Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144212024-02-03T17:28:58.832Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.999780595Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144222024-02-03T17:28:58.833Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:57.999886811Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144232024-02-03T17:28:58.833Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.000010949Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144242024-02-03T17:28:58.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.000207572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144252024-02-03T17:28:58.833Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.00024817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144262024-02-03T17:28:58.833Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.00027299Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144272024-02-03T17:28:58.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.000664237Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144282024-02-03T17:28:58.833Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.000741496Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144292024-02-03T17:28:58.833Z{"msg":"[0]R flush_numbers[0..12]: [242, 235, 211, 239, 239, 239, 240, 235, 243, 243, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.000850291Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144302024-02-03T17:28:58.833Z{"msg":"[0]R generation[0..12]: [132, 128, 116, 131, 131, 131, 132, 128, 132, 132, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.00090328Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144312024-02-03T17:28:58.833Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.0009309Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144322024-02-03T17:28:58.833Z{"msg":"[1]R flush_numbers[0..12]: [242, 235, 211, 239, 239, 239, 240, 235, 243, 243, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.000974716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144332024-02-03T17:28:58.833Z{"msg":"[1]R generation[0..12]: [132, 128, 116, 131, 131, 131, 132, 128, 132, 132, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.00100037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144342024-02-03T17:28:58.833Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001025758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144352024-02-03T17:28:58.833Z{"msg":"[2]R flush_numbers[0..12]: [242, 235, 211, 239, 239, 239, 240, 235, 243, 243, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001049745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144362024-02-03T17:28:58.834Z{"msg":"[2]R generation[0..12]: [132, 128, 116, 131, 131, 131, 132, 128, 132, 132, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001073178Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144372024-02-03T17:28:58.834Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001096501Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144382024-02-03T17:28:58.834Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001137424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144392024-02-03T17:28:58.834Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001161577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144402024-02-03T17:28:58.834Z{"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001201899Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144412024-02-03T17:28:58.834Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001225262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144422024-02-03T17:28:58.834Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
144432024-02-03T17:28:58.834Ztime":"2024-02-03T17:28:58.001267221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144442024-02-03T17:28:58.834Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.00129331Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144452024-02-03T17:28:58.834Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001345205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144462024-02-03T17:28:58.834Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001366924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144472024-02-03T17:28:58.834Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001388538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144482024-02-03T17:28:58.834Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001443662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144492024-02-03T17:28:58.834Z{"msg":"1d3025fa-2f01-47fb-94e9-12952ab769d7 is now active with session: 5025e229-b8b3-4f95-81f2-e14907ef0d58","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001467148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144502024-02-03T17:28:58.834Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001486571Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671,"session_id":"5025e229-b8b3-4f95-81f2-e14907ef0d58"}
144512024-02-03T17:28:58.834Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:28:58.001509047Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671}
144522024-02-03T17:28:58.834ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144532024-02-03T17:28:58.834ZLoading write count information from file "/tmp/test_repair_verify.data"
144542024-02-03T17:28:58.834ZRun Repair workload
144552024-02-03T17:28:58.834Z01/30 Write block 63 len 32768 data: 12 12 9 8 7 4 5 5
144562024-02-03T17:28:58.838Z02/30 Write block 319 len 4096 data: 11
144572024-02-03T17:28:58.838Z03/30 Write block 326 len 36864 data: 10 9 10 8 9 10 10 11 11
144582024-02-03T17:28:58.838Z04/30 Read block 17 len 8192
144592024-02-03T17:28:58.841Z05/30 Write block 371 len 32768 data: 7 6 5 6 7 10 11 12
144602024-02-03T17:28:58.841Z06/30 Read block 35 len 4096
144612024-02-03T17:28:58.841Z07/30 Read block 384 len 8192
144622024-02-03T17:28:58.849Z08/30 Read block 103 len 8192
144632024-02-03T17:28:58.849Z09/30 Read block 127 len 36864
144642024-02-03T17:28:58.849Z10/30 Read block 251 len 16384
144652024-02-03T17:28:58.849Z11/30 Read block 559 len 36864
144662024-02-03T17:28:58.852Z12/30 Read block 457 len 40960
144672024-02-03T17:28:58.852Z13/30 Write block 444 len 20480 data: 9 11 10 8 9
144682024-02-03T17:28:58.852Z14/30 Flush
144692024-02-03T17:28:58.855Z15/30 Flush
144702024-02-03T17:28:58.855Z16/30 Write block 353 len 16384 data: 10 10 13 17
144712024-02-03T17:28:58.855Z17/30 Read block 384 len 8192
144722024-02-03T17:28:58.858Z18/30 Write block 249 len 20480 data: 10 15 10 8 8
144732024-02-03T17:28:58.858Z19/30 Read block 281 len 32768
144742024-02-03T17:28:58.864Z20/30 Read block 323 len 40960
144752024-02-03T17:28:58.864Z21/30 Write block 455 len 40960 data: 7 11 14 14 15 12 10 12 12 11
144762024-02-03T17:28:58.864Z22/30 Read block 444 len 24576
144772024-02-03T17:28:58.867Z23/30 Write block 275 len 24576 data: 17 17 16 16 12 11
144782024-02-03T17:28:58.867Z24/30 Read block 342 len 40960
144792024-02-03T17:28:58.870Z25/30 Read block 160 len 8192
144802024-02-03T17:28:58.870Z26/30 Read block 119 len 12288
144812024-02-03T17:28:58.873Z27/30 Write block 469 len 16384 data: 8 8 6 5
144822024-02-03T17:28:58.873Z28/30 Write block 73 len 12288 data: 6 7 8
144832024-02-03T17:28:58.873Z29/30 Read block 66 len 28672
144842024-02-03T17:28:58.877Z30/30 Read block 504 len 12288
144852024-02-03T17:28:58.877Z----------------------------------------------------------------
144862024-02-03T17:28:58.877Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144872024-02-03T17:28:58.877ZStates: Active Active Active
144882024-02-03T17:28:58.877ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144892024-02-03T17:28:58.877Z 1 Acked 1000 Write 8 Done Done Done false
144902024-02-03T17:28:58.877Z 2 Acked 1001 Write 1 Sent Done Done false
144912024-02-03T17:28:58.877Z 3 Acked 1002 Write 9 Sent Done Done false
144922024-02-03T17:28:58.877Z 4 Acked 1003 Read 2 Sent Done Done false
144932024-02-03T17:28:58.878Z 5 Acked 1004 Write 8 Sent Done Done false
144942024-02-03T17:28:58.878Z 6 Acked 1005 Read 1 Sent Done Done false
144952024-02-03T17:28:58.878Z 7 Acked 1006 Read 2 Sent Done Done false
144962024-02-03T17:28:58.878Z 8 Acked 1007 Read 2 Sent Done Done false
144972024-02-03T17:28:58.878Z 9 Acked 1008 Read 9 Sent Done Done false
144982024-02-03T17:28:58.878Z 10 Acked 1009 Read 4 Sent Done Done false
144992024-02-03T17:28:58.878Z 11 Acked 1010 Read 9 Sent Done Done false
145002024-02-03T17:28:58.878Z 12 Acked 1011 Read 10 Sent Done Done false
145012024-02-03T17:28:58.878Z 13 Acked 1012 Write 5 Sent Done Done false
145022024-02-03T17:28:58.878Z 14 Acked 1013 Flush 0 Sent Done Done false
145032024-02-03T17:28:58.878Z 15 Acked 1014 Flush 0 Sent Done Done false
145042024-02-03T17:28:58.878Z 16 Acked 1015 Write 4 Sent Done Done false
145052024-02-03T17:28:58.878Z 17 Acked 1016 Read 2 Sent Done Done false
145062024-02-03T17:28:58.878Z 18 Acked 1017 Write 5 Sent Done Done false
145072024-02-03T17:28:58.878Z 19 Acked 1018 Read 8 Sent Done Done false
145082024-02-03T17:28:58.878Z 20 Acked 1019 Read 10 Sent Done Done false
145092024-02-03T17:28:58.878Z 21 Acked 1020 Write 10 Sent Done Done false
145102024-02-03T17:28:58.878Z 22 Acked 1021 Read 6 Sent Done Done false
145112024-02-03T17:28:58.878Z 23 Acked 1022 Write 6 Sent Done Done false
145122024-02-03T17:28:58.878Z 24 Acked 1023 Read 10 Sent Done Done false
145132024-02-03T17:28:58.878Z 25 Acked 1024 Read 2 Sent Done Done false
145142024-02-03T17:28:58.879Z 26 Acked 1025 Read 3 Sent Done Done false
145152024-02-03T17:28:58.879Z 27 Acked 1026 Write 4 Sent Done Done false
145162024-02-03T17:28:58.879Z 28 Acked 1027 Write 3 Sent Done Done false
145172024-02-03T17:28:58.879Z 29 Acked 1028 Read 7 Sent Done Done false
145182024-02-03T17:28:58.879Z 30 Acked 1029 Read 3 Sent Done Sent false
145192024-02-03T17:28:58.879Z STATES DS:0 DS:1 DS:2 TOTAL
145202024-02-03T17:28:58.879Z New 0 0 0 0
145212024-02-03T17:28:58.879Z Sent 29 0 1 30
145222024-02-03T17:28:58.879Z Done 1 30 29 60
145232024-02-03T17:28:58.879Z Skipped 0 0 0 0
145242024-02-03T17:28:58.879Z Error 0 0 0 0
145252024-02-03T17:28:58.879ZLast Flush: 0 1014 1014
145262024-02-03T17:28:58.879ZDownstairs last five completed:
145272024-02-03T17:28:58.879ZUpstairs last five completed: 30 29 28 27 26
145282024-02-03T17:28:58.879Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:28:58.04767006Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1671}
145292024-02-03T17:28:58.879ZWrote out file "/tmp/test_repair_verify.data"
145302024-02-03T17:28:58.902Z
145312024-02-03T17:28:58.913ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145322024-02-03T17:28:58.933ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145332024-02-03T17:28:58.933Z 0 000-019 132 132 132 242 242 242 F F F
145342024-02-03T17:28:58.933Z 1 020-039 128 128 128 235 235 235 F F F
145352024-02-03T17:28:58.933Z 2 040-059 116 116 116 211 211 211 F F F
145362024-02-03T17:28:58.933Z 3 060-079 131 134 134 239 246 246 T T T <---
145372024-02-03T17:28:58.933Z 4 080-099 131 131 131 239 239 239 F F F
145382024-02-03T17:28:58.933Z 5 100-119 131 131 131 239 239 239 F F F
145392024-02-03T17:28:58.933Z 6 120-139 132 132 132 240 240 240 F F F
145402024-02-03T17:28:58.933Z 7 140-159 128 128 128 235 235 235 F F F
145412024-02-03T17:28:58.933Z 8 160-179 132 132 132 243 243 243 F F F
145422024-02-03T17:28:58.933Z 9 180-199 132 132 132 243 243 243 F F F
145432024-02-03T17:28:58.933Z 10 200-219 131 131 131 239 239 239 F F F
145442024-02-03T17:28:58.934Z 11 220-239 132 132 132 240 240 240 F F F
145452024-02-03T17:28:58.934Z 12 240-259 133 133 133 245 245 245 F T T <---
145462024-02-03T17:28:58.934Z 13 260-279 133 133 133 245 245 245 F T T <---
145472024-02-03T17:28:58.934Z 14 280-299 127 127 127 231 231 231 F T T <---
145482024-02-03T17:28:58.934Z 15 300-319 132 134 134 240 246 246 F F F <---
145492024-02-03T17:28:58.934Z 16 320-339 131 134 134 239 246 246 F F F <---
145502024-02-03T17:28:58.934Z 17 340-359 132 132 132 241 241 241 F T T <---
145512024-02-03T17:28:58.934Z 18 360-379 123 134 134 224 246 246 F F F <---
145522024-02-03T17:28:58.934Z 19 380-399 131 131 131 239 239 239 F F F
145532024-02-03T17:28:58.934Z 20 400-419 130 130 130 238 238 238 F F F
145542024-02-03T17:28:58.934Z 21 420-439 132 132 132 243 243 243 F F F
145552024-02-03T17:28:58.934Z 22 440-459 132 134 134 240 246 246 F T T <---
145562024-02-03T17:28:58.935Z 23 460-479 132 132 132 240 240 240 F T T <---
145572024-02-03T17:28:58.935Z 24 480-499 124 124 124 225 225 225 F F F
145582024-02-03T17:28:58.935Z 25 500-519 131 131 131 239 239 239 F F F
145592024-02-03T17:28:58.935Z 26 520-539 124 124 124 225 225 225 F F F
145602024-02-03T17:28:58.935Z 27 540-559 132 132 132 240 240 240 F F F
145612024-02-03T17:28:58.935Z 28 560-579 122 122 122 223 223 223 F F F
145622024-02-03T17:28:58.935Z 29 580-599 133 133 133 245 245 245 F F F
145632024-02-03T17:28:58.935ZMax gen: 134, Max flush: 246
145642024-02-03T17:28:58.935ZError: Difference in extent metadata found!
145652024-02-03T17:28:58.935ZA Difference in extent metadata is expected here
145662024-02-03T17:28:58.935ZOn loop 67
145672024-02-03T17:28:58.935Z
145682024-02-03T17:28:58.935ZVerifying data now
145692024-02-03T17:29:03.828ZNo change after verify
145702024-02-03T17:29:03.829ZLoop: 67 Downstairs dump after verify (and repair):
145712024-02-03T17:29:03.856ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145722024-02-03T17:29:03.856Z 0 000-019 132 132 132 242 242 242 F F F
145732024-02-03T17:29:03.893Z 1 020-039 128 128 128 235 235 235 F F F
145742024-02-03T17:29:03.894Z 2 040-059 116 116 116 211 211 211 F F F
145752024-02-03T17:29:03.894Z 3 060-079 135 135 135 247 247 247 F F F
145762024-02-03T17:29:03.894Z 4 080-099 131 131 131 239 239 239 F F F
145772024-02-03T17:29:03.894Z 5 100-119 131 131 131 239 239 239 F F F
145782024-02-03T17:29:03.894Z 6 120-139 132 132 132 240 240 240 F F F
145792024-02-03T17:29:03.894Z 7 140-159 128 128 128 235 235 235 F F F
145802024-02-03T17:29:03.894Z 8 160-179 132 132 132 243 243 243 F F F
145812024-02-03T17:29:03.894Z 9 180-199 132 132 132 243 243 243 F F F
145822024-02-03T17:29:03.894Z 10 200-219 131 131 131 239 239 239 F F F
145832024-02-03T17:29:03.894Z 11 220-239 132 132 132 240 240 240 F F F
145842024-02-03T17:29:03.894Z 12 240-259 135 135 135 247 247 247 F F F
145852024-02-03T17:29:03.895Z 13 260-279 135 135 135 247 247 247 F F F
145862024-02-03T17:29:03.895Z 14 280-299 135 135 135 247 247 247 F F F
145872024-02-03T17:29:03.895Z 15 300-319 134 134 134 246 246 246 F F F
145882024-02-03T17:29:03.895Z 16 320-339 134 134 134 246 246 246 F F F
145892024-02-03T17:29:03.895Z 17 340-359 135 135 135 247 247 247 F F F
145902024-02-03T17:29:03.895Z 18 360-379 134 134 134 246 246 246 F F F
145912024-02-03T17:29:03.895Z 19 380-399 131 131 131 239 239 239 F F F
145922024-02-03T17:29:03.895Z 20 400-419 130 130 130 238 238 238 F F F
145932024-02-03T17:29:03.895Z 21 420-439 132 132 132 243 243 243 F F F
145942024-02-03T17:29:03.895Z 22 440-459 135 135 135 247 247 247 F F F
145952024-02-03T17:29:03.895Z 23 460-479 135 135 135 247 247 247 F F F
145962024-02-03T17:29:03.896Z 24 480-499 124 124 124 225 225 225 F F F
145972024-02-03T17:29:03.896Z 25 500-519 131 131 131 239 239 239 F F F
145982024-02-03T17:29:03.896Z 26 520-539 124 124 124 225 225 225 F F F
145992024-02-03T17:29:03.896Z 27 540-559 132 132 132 240 240 240 F F F
146002024-02-03T17:29:03.896Z 28 560-579 122 122 122 223 223 223 F F F
146012024-02-03T17:29:03.896Z 29 580-599 133 133 133 245 245 245 F F F
146022024-02-03T17:29:03.896ZMax gen: 135, Max flush: 247
146032024-02-03T17:29:03.896Z
146042024-02-03T17:29:03.896ZBegin loop 68
146052024-02-03T17:29:03.896ZDownstairs to restart: 0
146062024-02-03T17:29:03.906Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.063380857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679}
146072024-02-03T17:29:03.906Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.063862956Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679}
146082024-02-03T17:29:03.906Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":Crucible runtime is spawned
146092024-02-03T17:29:03.906Z30,"time":"2024-02-03T17:29:03.064138156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679}
146102024-02-03T17:29:03.906Z{"msg":"Crucible b79758cf-29ef-470a-bdb1-d3aab2e2597b has session id: 056796d6-11fe-4bb0-a846-030524bd7f2a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.064233759Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146112024-02-03T17:29:03.906Z{"msg":"Upstairs opts: Upstairs UUID: b79758cf-29ef-470a-bdb1-d3aab2e2597b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.06438426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146122024-02-03T17:29:03.906Z{"msg":"Crucible stats registered with UUID: b79758cf-29ef-470a-bdb1-d3aab2e2597b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.064533421Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146132024-02-03T17:29:03.906Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.064577427Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679}
146142024-02-03T17:29:03.906Z{"msg":"b79758cf-29ef-470a-bdb1-d3aab2e2597b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.064640798Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146152024-02-03T17:29:03.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.064682792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146162024-02-03T17:29:03.906Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.064862267Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146172024-02-03T17:29:03.906Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.064989236Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146182024-02-03T17:29:03.907Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065126543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146192024-02-03T17:29:03.907Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065199261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146202024-02-03T17:29:03.907Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065246603Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146212024-02-03T17:29:03.907Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065289711Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146222024-02-03T17:29:03.907Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065385484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146232024-02-03T17:29:03.907Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065420585Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146242024-02-03T17:29:03.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065477079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146252024-02-03T17:29:03.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065502455Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146262024-02-03T17:29:03.907Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.065525267Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146272024-02-03T17:29:03.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.066445558Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146282024-02-03T17:29:03.907Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.066514653Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146292024-02-03T17:29:03.907Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.066541305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146302024-02-03T17:29:03.907Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.066637895Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146312024-02-03T17:29:03.908Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.067337155Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146322024-02-03T17:29:03.908Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.067540988Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146332024-02-03T17:29:03.908Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.067725221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146342024-02-03T17:29:03.908Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.067816282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146352024-02-03T17:29:03.908Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.068250099Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146362024-02-03T17:29:03.908Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.068309682Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146372024-02-03T17:29:03.908Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.068375468Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146382024-02-03T17:29:03.908Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.068400492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146392024-02-03T17:29:03.908Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.06844282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146402024-02-03T17:29:03.908Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.068467783Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146412024-02-03T17:29:03.908Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.068749549Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146422024-02-03T17:29:03.909Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069441915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146432024-02-03T17:29:03.909Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069523329Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146442024-02-03T17:29:03.909Z{"msg":"[0]R flush_numbers[0..12]: [242, 235, 211, 247, 239, 239, 240, 235, 243, 243, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069581226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146452024-02-03T17:29:03.909Z{"msg":"[0]R generation[0..12]: [132, 128, 116, 135, 131, 131, 132, 128, 132, 132, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069608092Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146462024-02-03T17:29:03.909Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069632173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146472024-02-03T17:29:03.909Z{"msg":"[1]R flush_numbers[0..12]: [242, 235, 211, 247, 239, 239, 240, 235, 243, 243, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069655893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146482024-02-03T17:29:03.909Z{"msg":"[1]R generation[0..12]: [132, 128, 116, 135, 131, 131, 132, 128, 132, 132, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069678559Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146492024-02-03T17:29:03.909Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069721778Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146502024-02-03T17:29:03.909Z{"msg":"[2]R flush_numbers[0..12]: [242, 235, 211, 247, 239, 239, 240, 235, 243, 243, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069747221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146512024-02-03T17:29:03.909ZWait for a query_work_queue command to finish before sending IO
146522024-02-03T17:29:03.909Z{"msg":"[2]R generation[0..12]: [132, 128, 116, 135, 131, 131, 132, 128, 132, 132, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069794695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146532024-02-03T17:29:03.909Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069820046Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146542024-02-03T17:29:03.909Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069861052Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146552024-02-03T17:29:03.909Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069885276Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146562024-02-03T17:29:03.909Z{"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069925296Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146572024-02-03T17:29:03.909Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069948866Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146582024-02-03T17:29:03.909Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.069971667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146592024-02-03T17:29:03.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.070014599Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146602024-02-03T17:29:03.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.070038167Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146612024-02-03T17:29:03.909Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.070079702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146622024-02-03T17:29:03.910Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.070103336Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146632024-02-03T17:29:03.910Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.070144323Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146642024-02-03T17:29:03.910Z{"msg":"b79758cf-29ef-470a-bdb1-d3aab2e2597b is now active with session: 056796d6-11fe-4bb0-a846-030524bd7f2a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.07017527Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146652024-02-03T17:29:03.910Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.070197778Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679,"session_id":"056796d6-11fe-4bb0-a846-030524bd7f2a"}
146662024-02-03T17:29:03.910Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:03.070219344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679}
146672024-02-03T17:29:03.910ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146682024-02-03T17:29:03.910ZLoading write count information from file "/tmp/test_repair_verify.data"
146692024-02-03T17:29:03.910ZRun Repair workload
146702024-02-03T17:29:03.910Z01/30 Write block 391 len 16384 data: 8 7 8 9
146712024-02-03T17:29:03.910Z02/30 Read block 102 len 24576
146722024-02-03T17:29:03.920Z03/30 Write block 317 len 4096 data: 8
146732024-02-03T17:29:03.920Z04/30 Read block 305 len 20480
146742024-02-03T17:29:03.920Z05/30 Write block 40 len 28672 data: 8 7 8 9 8 6 5
146752024-02-03T17:29:03.920Z06/30 Write block 279 len 24576 data: 13 12 14 15 13 12
146762024-02-03T17:29:03.920Z07/30 Read block 285 len 28672
146772024-02-03T17:29:03.920Z08/30 Flush
146782024-02-03T17:29:03.920Z09/30 Flush
146792024-02-03T17:29:03.920Z10/30 Write block 426 len 36864 data: 8 8 8 5 9 10 9 7 9
146802024-02-03T17:29:03.920Z11/30 Write block 431 len 24576 data: 11 10 8 10 10 10
146812024-02-03T17:29:03.920Z12/30 Write block 319 len 24576 data: 12 9 9 10 9 9
146822024-02-03T17:29:03.928Z13/30 Write block 383 len 12288 data: 8 8 8
146832024-02-03T17:29:03.928Z14/30 Read block 156 len 16384
146842024-02-03T17:29:03.928Z15/30 Write block 506 len 8192 data: 21 21
146852024-02-03T17:29:03.928Z16/30 Read block 51 len 32768
146862024-02-03T17:29:03.928Z17/30 Write block 329 len 40960 data: 9 10 11 11 12 12 10 10 9 6
146872024-02-03T17:29:03.928Z18/30 Flush
146882024-02-03T17:29:03.938Z19/30 Write block 68 len 36864 data: 5 6 6 6 5 7 8 9 10
146892024-02-03T17:29:03.938Z20/30 Read block 342 len 12288
146902024-02-03T17:29:03.938Z21/30 Read block 297 len 40960
146912024-02-03T17:29:03.938Z22/30 Flush
146922024-02-03T17:29:03.949Z23/30 Write block 486 len 24576 data: 8 9 8 8 8 8
146932024-02-03T17:29:03.949Z24/30 Write block 500 len 28672 data: 15 17 15 16 16 20 22
146942024-02-03T17:29:03.949Z25/30 Write block 15 len 12288 data: 8 8 7
146952024-02-03T17:29:03.949Z26/30 Write block 458 len 4096 data: 15
146962024-02-03T17:29:03.949Z27/30 Write block 505 len 4096 data: 21
146972024-02-03T17:29:03.949Z28/30 Write block 195 len 20480 data: 7 9 7 8 9
146982024-02-03T17:29:03.949Z29/30 Write block 50 len 36864 data: 11 11 11 10 7 9 13 11 12
146992024-02-03T17:29:03.949Z30/30 Write block 171 len 4096 data: 10
147002024-02-03T17:29:03.949Z----------------------------------------------------------------
147012024-02-03T17:29:03.949Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:21
147022024-02-03T17:29:03.949ZStates: Active Active Active
147032024-02-03T17:29:03.949ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147042024-02-03T17:29:03.950Z 10 Acked 1009 Write 9 Sent Done Done false
147052024-02-03T17:29:03.950Z 11 Acked 1010 Write 6 Sent Done Done false
147062024-02-03T17:29:03.950Z 12 Acked 1011 Write 6 Sent Done Done false
147072024-02-03T17:29:03.950Z 13 Acked 1012 Write 3 Sent Done Done false
147082024-02-03T17:29:03.950Z 14 Acked 1013 Read 4 Sent Done Done false
147092024-02-03T17:29:03.950Z 15 Acked 1014 Write 2 Sent Done Done false
147102024-02-03T17:29:03.950Z 16 Acked 1015 Read 8 Sent Done Done false
147112024-02-03T17:29:03.950Z 17 Acked 1016 Write 10 Sent Done Done false
147122024-02-03T17:29:03.950Z 18 Acked 1017 Flush 0 Sent Done Done false
147132024-02-03T17:29:03.950Z 19 Acked 1018 Write 9 Sent Done Done false
147142024-02-03T17:29:03.950Z 20 Acked 1019 Read 3 Sent Done Done false
147152024-02-03T17:29:03.950Z 21 Acked 1020 Read 10 Sent Done Done false
147162024-02-03T17:29:03.950Z 22 Acked 1021 Flush 0 Sent Done Done false
147172024-02-03T17:29:03.950Z 23 Acked 1022 Write 6 Sent Done Done false
147182024-02-03T17:29:03.950Z 24 Acked 1023 Write 7 Sent Done Done false
147192024-02-03T17:29:03.950Z 25 Acked 1024 Write 3 Sent Done Done false
147202024-02-03T17:29:03.951Z 26 Acked 1025 Write 1 Sent Done Done false
147212024-02-03T17:29:03.951Z 27 Acked 1026 Write 1 Sent Done Done false
147222024-02-03T17:29:03.951Z 28 Acked 1027 Write 5 Sent Done Done false
147232024-02-03T17:29:03.951Z 29 Acked 1028 Write 9 Sent Sent Sent false
147242024-02-03T17:29:03.951Z 30 Acked 1029 Write 1 Sent Sent Sent false
147252024-02-03T17:29:03.951Z STATES DS:0 DS:1 DS:2 TOTAL
147262024-02-03T17:29:03.951Z New 0 0 0 0
147272024-02-03T17:29:03.951Z Sent 21 2 2 25
147282024-02-03T17:29:03.951Z Done 0 19 19 38
147292024-02-03T17:29:03.951Z Skipped 0 0 0 0
147302024-02-03T17:29:03.951Z Error 0 0 0 0
147312024-02-03T17:29:03.951ZLast Flush: 1008 1021 1021
147322024-02-03T17:29:03.951ZDownstairs last five completed: 1008 1007 1006 1005 1004
147332024-02-03T17:29:03.952ZUpstairs last five completed: 30 29 28 27 26
147342024-02-03T17:29:03.961Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:03.115996941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1679}
147352024-02-03T17:29:03.961ZWrote out file "/tmp/test_repair_verify.data"
147362024-02-03T17:29:03.961Z
147372024-02-03T17:29:03.976ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147382024-02-03T17:29:03.989ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147392024-02-03T17:29:03.989Z 0 000-019 132 132 132 242 242 242 F T T <---
147402024-02-03T17:29:03.990Z 1 020-039 128 128 128 235 235 235 F F F
147412024-02-03T17:29:03.990Z 2 040-059 136 136 136 248 248 248 F T T <---
147422024-02-03T17:29:03.990Z 3 060-079 135 136 136 247 251 251 F F F <---
147432024-02-03T17:29:04.007Z 4 080-099 131 131 131 239 239 239 F F F
147442024-02-03T17:29:04.007Z 5 100-119 131 131 131 239 239 239 F F F
147452024-02-03T17:29:04.007Z 6 120-139 132 132 132 240 240 240 F F F
147462024-02-03T17:29:04.008Z 7 140-159 128 128 128 235 235 235 F F F
147472024-02-03T17:29:04.008Z 8 160-179 132 132 132 243 243 243 F T T <---
147482024-02-03T17:29:04.008Z 9 180-199 132 132 132 243 243 243 F T T <---
147492024-02-03T17:29:04.008Z 10 200-219 131 131 131 239 239 239 F F F
147502024-02-03T17:29:04.008Z 11 220-239 132 132 132 240 240 240 F F F
147512024-02-03T17:29:04.008Z 12 240-259 135 135 135 247 247 247 F F F
147522024-02-03T17:29:04.008Z 13 260-279 136 136 136 248 248 248 F F F
147532024-02-03T17:29:04.009Z 14 280-299 136 136 136 248 248 248 F F F
147542024-02-03T17:29:04.009Z 15 300-319 136 136 136 248 250 250 F F F <---
147552024-02-03T17:29:04.009Z 16 320-339 134 136 136 246 250 250 F F F <---
147562024-02-03T17:29:04.009Z 17 340-359 135 135 135 247 247 247 F F F
147572024-02-03T17:29:04.009Z 18 360-379 134 134 134 246 246 246 F F F
147582024-02-03T17:29:04.009Z 19 380-399 136 136 136 248 250 250 F F F <---
147592024-02-03T17:29:04.009Z 20 400-419 130 130 130 238 238 238 F F F
147602024-02-03T17:29:04.009Z 21 420-439 132 136 136 243 250 250 F F F <---
147612024-02-03T17:29:04.009Z 22 440-459 135 135 135 247 247 247 F T T <---
147622024-02-03T17:29:04.009Z 23 460-479 135 135 135 247 247 247 F F F
147632024-02-03T17:29:04.009Z 24 480-499 124 124 124 225 225 225 F T T <---
147642024-02-03T17:29:04.010Z 25 500-519 131 136 136 239 250 250 F T T <---
147652024-02-03T17:29:04.010Z 26 520-539 124 124 124 225 225 225 F F F
147662024-02-03T17:29:04.010Z 27 540-559 132 132 132 240 240 240 F F F
147672024-02-03T17:29:04.010Z 28 560-579 122 122 122 223 223 223 F F F
147682024-02-03T17:29:04.010Z 29 580-599 133 133 133 245 245 245 F F F
147692024-02-03T17:29:04.010ZMax gen: 136, Max flush: 251
147702024-02-03T17:29:04.010ZError: Difference in extent metadata found!
147712024-02-03T17:29:04.010ZA Difference in extent metadata is expected here
147722024-02-03T17:29:04.010ZOn loop 68
147732024-02-03T17:29:04.010Z
147742024-02-03T17:29:04.016ZVerifying data now
147752024-02-03T17:29:09.073ZNo change after verify
147762024-02-03T17:29:09.073ZLoop: 68 Downstairs dump after verify (and repair):
147772024-02-03T17:29:09.097ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147782024-02-03T17:29:09.097Z 0 000-019 137 137 137 252 252 252 F F F
147792024-02-03T17:29:09.105Z 1 020-039 128 128 128 235 235 235 F F F
147802024-02-03T17:29:09.105Z 2 040-059 137 137 137 252 252 252 F F F
147812024-02-03T17:29:09.105Z 3 060-079 136 136 136 251 251 251 F F F
147822024-02-03T17:29:09.105Z 4 080-099 131 131 131 239 239 239 F F F
147832024-02-03T17:29:09.105Z 5 100-119 131 131 131 239 239 239 F F F
147842024-02-03T17:29:09.105Z 6 120-139 132 132 132 240 240 240 F F F
147852024-02-03T17:29:09.105Z 7 140-159 128 128 128 235 235 235 F F F
147862024-02-03T17:29:09.105Z 8 160-179 137 137 137 252 252 252 F F F
147872024-02-03T17:29:09.105Z 9 180-199 137 137 137 252 252 252 F F F
147882024-02-03T17:29:09.105Z 10 200-219 131 131 131 239 239 239 F F F
147892024-02-03T17:29:09.106Z 11 220-239 132 132 132 240 240 240 F F F
147902024-02-03T17:29:09.106Z 12 240-259 135 135 135 247 247 247 F F F
147912024-02-03T17:29:09.106Z 13 260-279 136 136 136 248 248 248 F F F
147922024-02-03T17:29:09.106Z 14 280-299 136 136 136 248 248 248 F F F
147932024-02-03T17:29:09.106Z 15 300-319 136 136 136 250 250 250 F F F
147942024-02-03T17:29:09.106Z 16 320-339 136 136 136 250 250 250 F F F
147952024-02-03T17:29:09.106Z 17 340-359 135 135 135 247 247 247 F F F
147962024-02-03T17:29:09.106Z 18 360-379 134 134 134 246 246 246 F F F
147972024-02-03T17:29:09.106Z 19 380-399 136 136 136 250 250 250 F F F
147982024-02-03T17:29:09.106Z 20 400-419 130 130 130 238 238 238 F F F
147992024-02-03T17:29:09.106Z 21 420-439 136 136 136 250 250 250 F F F
148002024-02-03T17:29:09.106Z 22 440-459 137 137 137 252 252 252 F F F
148012024-02-03T17:29:09.106Z 23 460-479 135 135 135 247 247 247 F F F
148022024-02-03T17:29:09.106Z 24 480-499 137 137 137 252 252 252 F F F
148032024-02-03T17:29:09.106Z 25 500-519 137 137 137 252 252 252 F F F
148042024-02-03T17:29:09.106Z 26 520-539 124 124 124 225 225 225 F F F
148052024-02-03T17:29:09.106Z 27 540-559 132 132 132 240 240 240 F F F
148062024-02-03T17:29:09.106Z 28 560-579 122 122 122 223 223 223 F F F
148072024-02-03T17:29:09.106Z 29 580-599 133 133 133 245 245 245 F F F
148082024-02-03T17:29:09.106ZMax gen: 137, Max flush: 252
148092024-02-03T17:29:09.112Z
148102024-02-03T17:29:09.112ZBegin loop 69
148112024-02-03T17:29:09.112ZDownstairs to restart: 2
148122024-02-03T17:29:09.138Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.309759887Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687}
148132024-02-03T17:29:09.142Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
148142024-02-03T17:29:09.142Z,"time":"2024-02-03T17:29:08.310435779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687}
148152024-02-03T17:29:09.142Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310560753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687}
148162024-02-03T17:29:09.142Z{"msg":"Crucible f9391bb2-a12a-48dd-adc1-fad81a698dfb has session id: 3999d69b-87a2-4bdb-960d-2d77c2b6d367","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310604901Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148172024-02-03T17:29:09.142Z{"msg":"Upstairs opts: Upstairs UUID: f9391bb2-a12a-48dd-adc1-fad81a698dfb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310658452Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148182024-02-03T17:29:09.142Z{"msg":"Crucible stats registered with UUID: f9391bb2-a12a-48dd-adc1-fad81a698dfb","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310726647Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148192024-02-03T17:29:09.142Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310763798Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687}
148202024-02-03T17:29:09.142Z{"msg":"f9391bb2-a12a-48dd-adc1-fad81a698dfb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310802021Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148212024-02-03T17:29:09.142Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310867435Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148222024-02-03T17:29:09.142Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310911206Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148232024-02-03T17:29:09.142Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.310979422Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148242024-02-03T17:29:09.142Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311024334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148252024-02-03T17:29:09.142Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311092862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148262024-02-03T17:29:09.143Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311134218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148272024-02-03T17:29:09.143Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311169232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148282024-02-03T17:29:09.143Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311233008Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148292024-02-03T17:29:09.143Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311273647Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148302024-02-03T17:29:09.143Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311365546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148312024-02-03T17:29:09.143Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311407071Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148322024-02-03T17:29:09.143Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.311600953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148332024-02-03T17:29:09.143Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.312985086Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148342024-02-03T17:29:09.143Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.313064502Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148352024-02-03T17:29:09.143Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.313110387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148362024-02-03T17:29:09.143Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.313151837Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148372024-02-03T17:29:09.211Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.314286697Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148382024-02-03T17:29:09.211Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.314353306Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148392024-02-03T17:29:09.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.314825372Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148402024-02-03T17:29:09.211Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.314934662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148412024-02-03T17:29:09.211Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.315388198Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148422024-02-03T17:29:09.211Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.315453256Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148432024-02-03T17:29:09.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.315480725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148442024-02-03T17:29:09.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.315746863Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148452024-02-03T17:29:09.212Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.315803674Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148462024-02-03T17:29:09.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.315829622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148472024-02-03T17:29:09.212Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.315853009Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148482024-02-03T17:29:09.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.316569916Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148492024-02-03T17:29:09.212Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.316702487Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148502024-02-03T17:29:09.212Z{"msg":"[0]R flush_numbers[0..12]: [252, 235, 252, 251, 239, 239, 240, 235, 252, 252, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.316775057Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148512024-02-03T17:29:09.212Z{"msg":"[0]R generation[0..12]: [137, 128, 137, 136, 131, 131, 132, 128, 137, 137, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.316820129Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148522024-02-03T17:29:09.212Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.316877115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148532024-02-03T17:29:09.212Z{"msg":"[1]R flush_numbers[0..12]: [252, 235, 252, 251, 239, 239, 240, 235, 252, 252, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.316922511Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148542024-02-03T17:29:09.212Z{"msg":"[1]R generation[0..12]: [137, 128, 137, 136, 131, 131, 132, 128, 137, 137, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.31698361Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}Wait for a query_work_queue command to finish before sending IO
148552024-02-03T17:29:09.212Z
148562024-02-03T17:29:09.212Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317044031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148572024-02-03T17:29:09.212Z{"msg":"[2]R flush_numbers[0..12]: [252, 235, 252, 251, 239, 239, 240, 235, 252, 252, 239, 240]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317103865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148582024-02-03T17:29:09.212Z{"msg":"[2]R generation[0..12]: [137, 128, 137, 136, 131, 131, 132, 128, 137, 137, 131, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317148927Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148592024-02-03T17:29:09.212Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317209846Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148602024-02-03T17:29:09.212Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317252384Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148612024-02-03T17:29:09.212Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317306677Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148622024-02-03T17:29:09.212Z{"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317347807Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148632024-02-03T17:29:09.212Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.31738817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148642024-02-03T17:29:09.212Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.31745086Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148652024-02-03T17:29:09.212Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317491102Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148662024-02-03T17:29:09.213Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317551319Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148672024-02-03T17:29:09.213Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317593469Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148682024-02-03T17:29:09.213Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.3176532ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148692024-02-03T17:29:09.213Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148702024-02-03T17:29:09.213Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317699709Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148712024-02-03T17:29:09.213Z{"msg":"f9391bb2-a12a-48dd-adc1-fad81a698dfb is now active with session: 3999d69b-87a2-4bdb-960d-2d77c2b6d367","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317753981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148722024-02-03T17:29:09.213Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.31779321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687,"session_id":"3999d69b-87a2-4bdb-960d-2d77c2b6d367"}
148732024-02-03T17:29:09.213Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:08.317845983Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687}
148742024-02-03T17:29:09.213ZLoading write count information from file "/tmp/test_repair_verify.data"
148752024-02-03T17:29:09.213ZRun Repair workload
148762024-02-03T17:29:09.213Z01/30 Write block 511 len 12288 data: 11 10 8
148772024-02-03T17:29:09.213Z02/30 Write block 280 len 40960 data: 13 15 16 14 13 10 9 10 10 9
148782024-02-03T17:29:09.213Z03/30 Read block 531 len 8192
148792024-02-03T17:29:09.213Z04/30 Read block 443 len 4096
148802024-02-03T17:29:09.213Z05/30 Flush
148812024-02-03T17:29:09.213Z06/30 Write block 227 len 8192 data: 18 15
148822024-02-03T17:29:09.213Z07/30 Read block 279 len 4096
148832024-02-03T17:29:09.213Z08/30 Write block 388 len 32768 data: 8 9 9 9 8 9 10 11
148842024-02-03T17:29:09.213Z09/30 Write block 488 len 8192 data: 9 9
148852024-02-03T17:29:09.213Z10/30 Write block 423 len 20480 data: 13 12 10 9 9
148862024-02-03T17:29:09.213Z11/30 Read block 217 len 36864
148872024-02-03T17:29:09.213Z12/30 Read block 44 len 12288
148882024-02-03T17:29:09.213Z13/30 Write block 599 len 4096 data: 5
148892024-02-03T17:29:09.213Z14/30 Write block 17 len 12288 data: 8 7 7
148902024-02-03T17:29:09.213Z15/30 Read block 274 len 28672
148912024-02-03T17:29:09.214Z16/30 Read block 546 len 28672
148922024-02-03T17:29:09.214Z17/30 Flush
148932024-02-03T17:29:09.214Z18/30 Write block 342 len 40960 data: 8 8 9 9 8 10 12 11 10 10
148942024-02-03T17:29:09.214Z19/30 Write block 494 len 12288 data: 11 10 11
148952024-02-03T17:29:09.214Z20/30 Write block 103 len 36864 data: 12 10 13 10 11 12 11 12 11
148962024-02-03T17:29:09.214Z21/30 Read block 391 len 24576
148972024-02-03T17:29:09.214Z22/30 Flush
148982024-02-03T17:29:09.214Z23/30 Write block 580 len 32768 data: 15 15 14 11 12 10 14 11
148992024-02-03T17:29:09.214Z24/30 Write block 491 len 16384 data: 9 9 12 12
149002024-02-03T17:29:09.214Z25/30 Write block 57 len 12288 data: 12 13 13
149012024-02-03T17:29:09.214Z26/30 Read block 67 len 28672
149022024-02-03T17:29:09.214Z27/30 Write block 509 len 24576 data: 18 14 12 11 9 6
149032024-02-03T17:29:09.214Z28/30 Read block 252 len 24576
149042024-02-03T17:29:09.214Z29/30 Read block 184 len 12288
149052024-02-03T17:29:09.214Z30/30 Write block 4 len 20480 data: 9 11 11 13 13
149062024-02-03T17:29:09.214Z----------------------------------------------------------------
149072024-02-03T17:29:09.214Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
149082024-02-03T17:29:09.214ZStates: Active Active Active
149092024-02-03T17:29:09.214ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149102024-02-03T17:29:09.214Z 1 Acked 1000 Write 3 Done Done Done false
149112024-02-03T17:29:09.214Z 2 Acked 1001 Write 10 Done Done Done false
149122024-02-03T17:29:09.214Z 3 Acked 1002 Read 2 Done Done Done false
149132024-02-03T17:29:09.214Z 4 Acked 1003 Read 1 Done Done Sent false
149142024-02-03T17:29:09.214Z 5 Acked 1004 Flush 0 Done Done Sent false
149152024-02-03T17:29:09.215Z 6 Acked 1005 Write 2 Done Done Sent false
149162024-02-03T17:29:09.215Z 7 Acked 1006 Read 1 Done Done Sent false
149172024-02-03T17:29:09.215Z 8 Acked 1007 Write 8 Done Done Sent false
149182024-02-03T17:29:09.215Z 9 Acked 1008 Write 2 Done Done Sent false
149192024-02-03T17:29:09.215Z 10 Acked 1009 Write 5 Done Done Sent false
149202024-02-03T17:29:09.215Z 11 Acked 1010 Read 9 Done Done Sent false
149212024-02-03T17:29:09.215Z 12 Acked 1011 Read 3 Done Done Sent false
149222024-02-03T17:29:09.215Z 13 Acked 1012 Write 1 Done Done Sent false
149232024-02-03T17:29:09.215Z 14 Acked 1013 Write 3 Done Done Sent false
149242024-02-03T17:29:09.215Z 15 Acked 1014 Read 7 Done Done Sent false
149252024-02-03T17:29:09.215Z 16 Acked 1015 Read 7 Done Done Sent false
149262024-02-03T17:29:09.215Z 17 Acked 1016 Flush 0 Done Done Sent false
149272024-02-03T17:29:09.215Z 18 Acked 1017 Write 10 Done Done Sent false
149282024-02-03T17:29:09.215Z 19 Acked 1018 Write 3 Done Done Sent false
149292024-02-03T17:29:09.215Z 20 Acked 1019 Write 9 Done Done Sent false
149302024-02-03T17:29:09.215Z 21 Acked 1020 Read 6 Done Done Sent false
149312024-02-03T17:29:09.215Z 22 Acked 1021 Flush 0 Done Done Sent false
149322024-02-03T17:29:09.215Z 23 Acked 1022 Write 8 Done Done Sent false
149332024-02-03T17:29:09.215Z 24 Acked 1023 Write 4 Done Done Sent false
149342024-02-03T17:29:09.215Z 25 Acked 1024 Write 3 Done Done Sent false
149352024-02-03T17:29:09.215Z 26 Acked 1025 Read 7 Done Done Sent false
149362024-02-03T17:29:09.215Z 27 Acked 1026 Write 6 Done Done Sent false
149372024-02-03T17:29:09.221Z 28 Acked 1027 Read 6 Done Done Sent false
149382024-02-03T17:29:09.221Z 29 Acked 1028 Read 3 Done Done Sent false
149392024-02-03T17:29:09.221Z 30 Acked 1029 Write 5 Sent Sent Sent false
149402024-02-03T17:29:09.221Z STATES DS:0 DS:1 DS:2 TOTAL
149412024-02-03T17:29:09.221Z New 0 0 0 0
149422024-02-03T17:29:09.221Z Sent 1 1 27 29
149432024-02-03T17:29:09.221Z Done 29 29 3 61
149442024-02-03T17:29:09.221Z Skipped 0 0 0 0
149452024-02-03T17:29:09.221Z Error 0 0 0 0
149462024-02-03T17:29:09.221ZLast Flush: 1021 1021 0
149472024-02-03T17:29:09.221ZDownstairs last five completed:
149482024-02-03T17:29:09.221ZUpstairs last five completed: 30 29 28 27 26
149492024-02-03T17:29:09.221Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:08.363869638Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1687}
149502024-02-03T17:29:09.221ZWrote out file "/tmp/test_repair_verify.data"
149512024-02-03T17:29:09.221Z
149522024-02-03T17:29:09.221ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149532024-02-03T17:29:09.235ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149542024-02-03T17:29:09.235Z 0 000-019 138 138 137 254 254 252 T T F <---
149552024-02-03T17:29:09.239Z 1 020-039 128 128 128 235 235 235 F F F
149562024-02-03T17:29:09.239Z 2 040-059 137 137 137 252 252 252 T T F <---
149572024-02-03T17:29:09.239Z 3 060-079 136 136 136 251 251 251 F F F
149582024-02-03T17:29:09.239Z 4 080-099 131 131 131 239 239 239 F F F
149592024-02-03T17:29:09.239Z 5 100-119 138 138 131 255 255 239 F F F <---
149602024-02-03T17:29:09.239Z 6 120-139 132 132 132 240 240 240 F F F
149612024-02-03T17:29:09.239Z 7 140-159 128 128 128 235 235 235 F F F
149622024-02-03T17:29:09.239Z 8 160-179 137 137 137 252 252 252 F F F
149632024-02-03T17:29:09.239Z 9 180-199 137 137 137 252 252 252 F F F
149642024-02-03T17:29:09.239Z 10 200-219 131 131 131 239 239 239 F F F
149652024-02-03T17:29:09.239Z 11 220-239 138 138 132 254 254 240 F F F <---
149662024-02-03T17:29:09.239Z 12 240-259 135 135 135 247 247 247 F F F
149672024-02-03T17:29:09.239Z 13 260-279 136 136 136 248 248 248 F F F
149682024-02-03T17:29:09.239Z 14 280-299 138 138 136 253 253 248 F F T <---
149692024-02-03T17:29:09.239Z 15 300-319 136 136 136 250 250 250 F F F
149702024-02-03T17:29:09.239Z 16 320-339 136 136 136 250 250 250 F F F
149712024-02-03T17:29:09.239Z 17 340-359 138 138 135 255 255 247 F F F <---
149722024-02-03T17:29:09.239Z 18 360-379 134 134 134 246 246 246 F F F
149732024-02-03T17:29:09.239Z 19 380-399 138 138 136 254 254 250 F F F <---
149742024-02-03T17:29:09.239Z 20 400-419 130 130 130 238 238 238 F F F
149752024-02-03T17:29:09.240Z 21 420-439 138 138 136 254 254 250 F F F <---
149762024-02-03T17:29:09.240Z 22 440-459 137 137 137 252 252 252 F F F
149772024-02-03T17:29:09.240Z 23 460-479 135 135 135 247 247 247 F F F
149782024-02-03T17:29:09.240Z 24 480-499 138 138 137 255 255 252 T T F <---
149792024-02-03T17:29:09.240Z 25 500-519 138 138 137 253 253 252 T T T <---
149802024-02-03T17:29:09.240Z 26 520-539 124 124 124 225 225 225 F F F
149812024-02-03T17:29:09.240Z 27 540-559 132 132 132 240 240 240 F F F
149822024-02-03T17:29:09.240Z 28 560-579 122 122 122 223 223 223 F F F
149832024-02-03T17:29:09.240Z 29 580-599 138 138 133 254 254 245 T T F <---
149842024-02-03T17:29:09.240ZMax gen: 138, Max flush: 255
149852024-02-03T17:29:09.240ZError: Difference in extent metadata found!
149862024-02-03T17:29:09.244ZA Difference in extent metadata is expected here
149872024-02-03T17:29:09.244ZOn loop 69
149882024-02-03T17:29:09.244Z
149892024-02-03T17:29:09.247ZVerifying data now
149902024-02-03T17:29:14.251ZNo change after verify
149912024-02-03T17:29:14.251ZLoop: 69 Downstairs dump after verify (and repair):
149922024-02-03T17:29:14.275ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149932024-02-03T17:29:14.275Z 0 000-019 139 139 139 256 256 256 F F F
149942024-02-03T17:29:14.275Z 1 020-039 128 128 128 235 235 235 F F F
149952024-02-03T17:29:14.275Z 2 040-059 139 139 139 256 256 256 F F F
149962024-02-03T17:29:14.275Z 3 060-079 136 136 136 251 251 251 F F F
149972024-02-03T17:29:14.279Z 4 080-099 131 131 131 239 239 239 F F F
149982024-02-03T17:29:14.279Z 5 100-119 138 138 138 255 255 255 F F F
149992024-02-03T17:29:14.279Z 6 120-139 132 132 132 240 240 240 F F F
150002024-02-03T17:29:14.280Z 7 140-159 128 128 128 235 235 235 F F F
150012024-02-03T17:29:14.280Z 8 160-179 137 137 137 252 252 252 F F F
150022024-02-03T17:29:14.280Z 9 180-199 137 137 137 252 252 252 F F F
150032024-02-03T17:29:14.280Z 10 200-219 131 131 131 239 239 239 F F F
150042024-02-03T17:29:14.280Z 11 220-239 138 138 138 254 254 254 F F F
150052024-02-03T17:29:14.280Z 12 240-259 135 135 135 247 247 247 F F F
150062024-02-03T17:29:14.280Z 13 260-279 136 136 136 248 248 248 F F F
150072024-02-03T17:29:14.280Z 14 280-299 138 138 138 253 253 253 F F F
150082024-02-03T17:29:14.280Z 15 300-319 136 136 136 250 250 250 F F F
150092024-02-03T17:29:14.280Z 16 320-339 136 136 136 250 250 250 F F F
150102024-02-03T17:29:14.280Z 17 340-359 138 138 138 255 255 255 F F F
150112024-02-03T17:29:14.280Z 18 360-379 134 134 134 246 246 246 F F F
150122024-02-03T17:29:14.280Z 19 380-399 138 138 138 254 254 254 F F F
150132024-02-03T17:29:14.280Z 20 400-419 130 130 130 238 238 238 F F F
150142024-02-03T17:29:14.280Z 21 420-439 138 138 138 254 254 254 F F F
150152024-02-03T17:29:14.280Z 22 440-459 137 137 137 252 252 252 F F F
150162024-02-03T17:29:14.280Z 23 460-479 135 135 135 247 247 247 F F F
150172024-02-03T17:29:14.280Z 24 480-499 139 139 139 256 256 256 F F F
150182024-02-03T17:29:14.280Z 25 500-519 139 139 139 256 256 256 F F F
150192024-02-03T17:29:14.280Z 26 520-539 124 124 124 225 225 225 F F F
150202024-02-03T17:29:14.280Z 27 540-559 132 132 132 240 240 240 F F F
150212024-02-03T17:29:14.280Z 28 560-579 122 122 122 223 223 223 F F F
150222024-02-03T17:29:14.281Z 29 580-599 139 139 139 256 256 256 F F F
150232024-02-03T17:29:14.281ZMax gen: 139, Max flush: 256
150242024-02-03T17:29:14.286Z
150252024-02-03T17:29:14.286ZBegin loop 70
150262024-02-03T17:29:14.286ZDownstairs to restart: 1
150272024-02-03T17:29:14.322Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.486618285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695}
150282024-02-03T17:29:14.323Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.48712471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695}
150292024-02-03T17:29:14.358Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
150302024-02-03T17:29:14.359Z,"time":"2024-02-03T17:29:13.487239779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695}
150312024-02-03T17:29:14.359Z{"msg":"Crucible 39ff034c-3edb-4215-b916-90885621ed9d has session id: 7f74ba45-1e8d-4f54-86ff-c403b81dbede","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.487329853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150322024-02-03T17:29:14.359Z{"msg":"Upstairs opts: Upstairs UUID: 39ff034c-3edb-4215-b916-90885621ed9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.487393617Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150332024-02-03T17:29:14.359Z{"msg":"Crucible stats registered with UUID: 39ff034c-3edb-4215-b916-90885621ed9d","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.487451378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150342024-02-03T17:29:14.359Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.487491909Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695}
150352024-02-03T17:29:14.359Z{"msg":"39ff034c-3edb-4215-b916-90885621ed9d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.487663018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150362024-02-03T17:29:14.359Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.487726541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150372024-02-03T17:29:14.359Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.487842095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150382024-02-03T17:29:14.359Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.48792571Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150392024-02-03T17:29:14.359Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488075237Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150402024-02-03T17:29:14.359Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488220105Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150412024-02-03T17:29:14.359Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488384587Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150422024-02-03T17:29:14.359Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488454766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150432024-02-03T17:29:14.359Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488484306Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150442024-02-03T17:29:14.359Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488547606Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150452024-02-03T17:29:14.359Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488678973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150462024-02-03T17:29:14.360Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488728679Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150472024-02-03T17:29:14.360Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:13.488796772Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150482024-02-03T17:29:14.360Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:13.488865546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150492024-02-03T17:29:14.360Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:13.488898046Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150502024-02-03T17:29:14.360Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.488921942Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150512024-02-03T17:29:14.360Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.48899347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150522024-02-03T17:29:14.360Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.489339161Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150532024-02-03T17:29:14.360Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.489859041Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150542024-02-03T17:29:14.360Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.489942653Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150552024-02-03T17:29:14.360Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.490058655Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150562024-02-03T17:29:14.360Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.490196685Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150572024-02-03T17:29:14.360Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.490918614Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150582024-02-03T17:29:14.360Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.491131098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150592024-02-03T17:29:14.360Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.491784233Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150602024-02-03T17:29:14.360Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.491846553Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150612024-02-03T17:29:14.360Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.491873155Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150622024-02-03T17:29:14.360Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.491894035Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150632024-02-03T17:29:14.360Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.491935185Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150642024-02-03T17:29:14.360Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:13.491959503Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150652024-02-03T17:29:15.321Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.490036075Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150662024-02-03T17:29:15.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.490307246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150672024-02-03T17:29:15.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.491515126Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150682024-02-03T17:29:15.340Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.491584151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150692024-02-03T17:29:15.340Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.49230521Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150702024-02-03T17:29:15.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493175715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150712024-02-03T17:29:15.340Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493236565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150722024-02-03T17:29:15.340Z{"msg":"[0]R flush_numbers[0..12]: [256, 235, 256, 251, 239, 255, 240, 235, 252, 252, 239, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493264045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150732024-02-03T17:29:15.340Z{"msg":"[0]R generation[0..12]: [139, 128, 139, 136, 131, 138, 132, 128, 137, 137, 131, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493287976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150742024-02-03T17:29:15.341Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493331578Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150752024-02-03T17:29:15.341Z{"msg":"[1]R flush_numbers[0..12]: [256, 235, 256, 251, 239, 255, 240, 235, 252, 252, 239, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493358017Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150762024-02-03T17:29:15.341Z{"msg":"[1]R generation[0..12]: [139, 128, 139, 136, 131, 138, 132, 128, 137, 137, 131, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493400149Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150772024-02-03T17:29:15.341Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493426305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150782024-02-03T17:29:15.341Z{"msg":"[2]R flush_numbers[0..12]: [256, 235, 256, 251, 239, 255, 240, 235, 252, 252, 239, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493468342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150792024-02-03T17:29:15.341Z{"msg":"[2]R generation[0..12]: [139, 128, 139, 136, 131, 138, 132, 128, 137, 137, 131, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493493467Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150802024-02-03T17:29:15.341Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493537663Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150812024-02-03T17:29:15.341Z{"msg":"Max found gen is 140"Wait for a query_work_queue command to finish before sending IO
150822024-02-03T17:29:15.341Z,"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493565596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150832024-02-03T17:29:15.341Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493611577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150842024-02-03T17:29:15.342Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493642052Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150852024-02-03T17:29:15.342Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.49371798Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150862024-02-03T17:29:15.342Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493742897Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150872024-02-03T17:29:15.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493785796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150882024-02-03T17:29:15.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493809372Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150892024-02-03T17:29:15.342Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493864476Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150902024-02-03T17:29:15.342Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493899897Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150912024-02-03T17:29:15.342Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493921153Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150922024-02-03T17:29:15.342Z{"msg":"39ff034c-3edb-4215-b916-90885621ed9d is now active with session: 7f74ba45-1e8d-4f54-86ff-c403b81dbede","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493969184Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150932024-02-03T17:29:15.343Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.493990454Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695,"session_id":"7f74ba45-1e8d-4f54-86ff-c403b81dbede"}
150942024-02-03T17:29:15.343Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:14.49403978Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695}
150952024-02-03T17:29:15.343ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150962024-02-03T17:29:15.343ZLoading write count information from file "/tmp/test_repair_verify.data"
150972024-02-03T17:29:15.343ZRun Repair workload
150982024-02-03T17:29:15.343Z01/30 Write block 82 len 12288 data: 9 13 14
150992024-02-03T17:29:15.343Z02/30 Write block 12 len 20480 data: 15 12 10 9 9
151002024-02-03T17:29:15.343Z03/30 Write block 390 len 12288 data: 10 10 9
151012024-02-03T17:29:15.343Z04/30 Read block 189 len 24576
151022024-02-03T17:29:15.358Z05/30 Write block 128 len 32768 data: 14 16 17 17 17 12 15 17
151032024-02-03T17:29:15.358Z06/30 Flush
151042024-02-03T17:29:15.359Z07/30 Write block 300 len 4096 data: 7
151052024-02-03T17:29:15.359Z08/30 Read block 0 len 28672
151062024-02-03T17:29:15.359Z09/30 Write block 517 len 8192 data: 9 8
151072024-02-03T17:29:15.359Z10/30 Read block 560 len 28672
151082024-02-03T17:29:15.360Z11/30 Write block 368 len 20480 data: 8 7 6 8 7
151092024-02-03T17:29:15.360Z12/30 Read block 266 len 32768
151102024-02-03T17:29:15.360Z13/30 Flush
151112024-02-03T17:29:15.360Z14/30 Write block 319 len 24576 data: 13 10 10 11 10 10
151122024-02-03T17:29:15.361Z15/30 Write block 531 len 36864 data: 10 10 7 6 9 10 10 9 9
151132024-02-03T17:29:15.361Z16/30 Read block 445 len 12288
151142024-02-03T17:29:15.361Z17/30 Flush
151152024-02-03T17:29:15.369Z18/30 Write block 396 len 40960 data: 11 8 7 8 9 6 7 10 9 14
151162024-02-03T17:29:15.369Z19/30 Write block 453 len 16384 data: 9 7 8 12
151172024-02-03T17:29:15.369Z20/30 Read block 167 len 20480
151182024-02-03T17:29:15.369Z21/30 Write block 556 len 24576 data: 10 9 10 11 7 5
151192024-02-03T17:29:15.369Z22/30 Write block 24 len 4096 data: 11
151202024-02-03T17:29:15.369Z23/30 Read block 99 len 16384
151212024-02-03T17:29:15.369Z24/30 Read block 295 len 8192
151222024-02-03T17:29:15.369Z25/30 Read block 224 len 24576
151232024-02-03T17:29:15.369Z26/30 Write block 575 len 40960 data: 9 11 12 15 15 16 16 15 12 13
151242024-02-03T17:29:15.369Z27/30 Write block 42 len 4096 data: 9
151252024-02-03T17:29:15.369Z28/30 Read block 132 len 28672
151262024-02-03T17:29:15.374Z29/30 Write block 401 len 4096 data: 7
151272024-02-03T17:29:15.374Z30/30 Read block 405 len 8192
151282024-02-03T17:29:15.374Z----------------------------------------------------------------
151292024-02-03T17:29:15.374Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:24
151302024-02-03T17:29:15.374ZStates: Active Active Active
151312024-02-03T17:29:15.374ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151322024-02-03T17:29:15.374Z 7 Acked 1006 Write 1 Done Done Done false
151332024-02-03T17:29:15.374Z 8 Acked 1007 Read 7 Done Done Done false
151342024-02-03T17:29:15.374Z 9 Acked 1008 Write 2 Done Done Done false
151352024-02-03T17:29:15.374Z 10 Acked 1009 Read 7 Done Sent Done false
151362024-02-03T17:29:15.375Z 11 Acked 1010 Write 5 Done Sent Done false
151372024-02-03T17:29:15.375Z 12 Acked 1011 Read 8 Done Sent Done false
151382024-02-03T17:29:15.375Z 13 Acked 1012 Flush 0 Done Sent Done false
151392024-02-03T17:29:15.375Z 14 Acked 1013 Write 6 Done Sent Done false
151402024-02-03T17:29:15.375Z 15 Acked 1014 Write 9 Done Sent Done false
151412024-02-03T17:29:15.375Z 16 Acked 1015 Read 3 Done Sent Done false
151422024-02-03T17:29:15.375Z 17 Acked 1016 Flush 0 Done Sent Done false
151432024-02-03T17:29:15.375Z 18 Acked 1017 Write 10 Done Sent Done false
151442024-02-03T17:29:15.375Z 19 Acked 1018 Write 4 Done Sent Done false
151452024-02-03T17:29:15.375Z 20 Acked 1019 Read 5 Done Sent Done false
151462024-02-03T17:29:15.375Z 21 Acked 1020 Write 6 Done Sent Done false
151472024-02-03T17:29:15.375Z 22 Acked 1021 Write 1 Done Sent Done false
151482024-02-03T17:29:15.376Z 23 Acked 1022 Read 4 Done Sent Done false
151492024-02-03T17:29:15.376Z 24 Acked 1023 Read 2 Done Sent Done false
151502024-02-03T17:29:15.376Z 25 Acked 1024 Read 6 Done Sent Done false
151512024-02-03T17:29:15.376Z 26 Acked 1025 Write 10 Done Sent Done false
151522024-02-03T17:29:15.376Z 27 Acked 1026 Write 1 Done Sent Done false
151532024-02-03T17:29:15.376Z 28 Acked 1027 Read 7 Done Sent Done false
151542024-02-03T17:29:15.376Z 29 Acked 1028 Write 1 Done Sent Done false
151552024-02-03T17:29:15.376Z 30 Acked 1029 Read 2 Done Sent Sent false
151562024-02-03T17:29:15.376Z STATES DS:0 DS:1 DS:2 TOTAL
151572024-02-03T17:29:15.376Z New 0 0 0 0
151582024-02-03T17:29:15.376Z Sent 0 21 1 22
151592024-02-03T17:29:15.376Z Done 24 3 23 50
151602024-02-03T17:29:15.376Z Skipped 0 0 0 0
151612024-02-03T17:29:15.377Z Error 0 0 0 0
151622024-02-03T17:29:15.377ZLast Flush: 1016 1005 1016
151632024-02-03T17:29:15.377ZDownstairs last five completed: 1005 1004 1003 1002 1001
151642024-02-03T17:29:15.377ZUpstairs last five completed: 30 29 28 27 26
151652024-02-03T17:29:15.377Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:14.54120126Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1695}
151662024-02-03T17:29:15.377ZWrote out file "/tmp/test_repair_verify.data"
151672024-02-03T17:29:15.381Z
151682024-02-03T17:29:15.384ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151692024-02-03T17:29:15.413ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151702024-02-03T17:29:15.413Z 0 000-019 140 140 140 257 257 257 F F F
151712024-02-03T17:29:15.417Z 1 020-039 128 128 128 235 235 235 T F T <---
151722024-02-03T17:29:15.417Z 2 040-059 139 139 139 256 256 256 T F T <---
151732024-02-03T17:29:15.418Z 3 060-079 136 136 136 251 251 251 F F F
151742024-02-03T17:29:15.418Z 4 080-099 140 140 140 257 257 257 F F F
151752024-02-03T17:29:15.418Z 5 100-119 138 138 138 255 255 255 F F F
151762024-02-03T17:29:15.418Z 6 120-139 140 140 140 257 257 257 F F F
151772024-02-03T17:29:15.418Z 7 140-159 128 128 128 235 235 235 F F F
151782024-02-03T17:29:15.418Z 8 160-179 137 137 137 252 252 252 F F F
151792024-02-03T17:29:15.418Z 9 180-199 137 137 137 252 252 252 F F F
151802024-02-03T17:29:15.418Z 10 200-219 131 131 131 239 239 239 F F F
151812024-02-03T17:29:15.418Z 11 220-239 138 138 138 254 254 254 F F F
151822024-02-03T17:29:15.418Z 12 240-259 135 135 135 247 247 247 F F F
151832024-02-03T17:29:15.418Z 13 260-279 136 136 136 248 248 248 F F F
151842024-02-03T17:29:15.418Z 14 280-299 138 138 138 253 253 253 F F F
151852024-02-03T17:29:15.419Z 15 300-319 140 136 140 259 250 259 F T F <---
151862024-02-03T17:29:15.419Z 16 320-339 140 136 140 259 250 259 F F F <---
151872024-02-03T17:29:15.419Z 17 340-359 138 138 138 255 255 255 F F F
151882024-02-03T17:29:15.419Z 18 360-379 140 134 140 258 246 258 F F F <---
151892024-02-03T17:29:15.419Z 19 380-399 140 140 140 257 257 257 T F T <---
151902024-02-03T17:29:15.419Z 20 400-419 130 130 130 238 238 238 T F T <---
151912024-02-03T17:29:15.419Z 21 420-439 138 138 138 254 254 254 F F F
151922024-02-03T17:29:15.419Z 22 440-459 137 137 137 252 252 252 T F T <---
151932024-02-03T17:29:15.419Z 23 460-479 135 135 135 247 247 247 F F F
151942024-02-03T17:29:15.419Z 24 480-499 139 139 139 256 256 256 F F F
151952024-02-03T17:29:15.419Z 25 500-519 140 139 140 258 256 258 F T F <---
151962024-02-03T17:29:15.419Z 26 520-539 140 124 140 259 225 259 F F F <---
151972024-02-03T17:29:15.419Z 27 540-559 132 132 132 240 240 240 T F T <---
151982024-02-03T17:29:15.420Z 28 560-579 122 122 122 223 223 223 T F T <---
151992024-02-03T17:29:15.420Z 29 580-599 139 139 139 256 256 256 T F T <---
152002024-02-03T17:29:15.420ZMax gen: 140, Max flush: 259
152012024-02-03T17:29:15.420ZError: Difference in extent metadata found!
152022024-02-03T17:29:15.424ZA Difference in extent metadata is expected here
152032024-02-03T17:29:15.424ZOn loop 70
152042024-02-03T17:29:15.424Z
152052024-02-03T17:29:15.424ZVerifying data now
152062024-02-03T17:29:20.598ZNo change after verify
152072024-02-03T17:29:20.598ZLoop: 70 Downstairs dump after verify (and repair):
152082024-02-03T17:29:20.623ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152092024-02-03T17:29:20.623Z 0 000-019 140 140 140 257 257 257 F F F
152102024-02-03T17:29:20.623Z 1 020-039 141 141 141 260 260 260 F F F
152112024-02-03T17:29:20.630Z 2 040-059 141 141 141 260 260 260 F F F
152122024-02-03T17:29:20.630Z 3 060-079 136 136 136 251 251 251 F F F
152132024-02-03T17:29:20.631Z 4 080-099 140 140 140 257 257 257 F F F
152142024-02-03T17:29:20.631Z 5 100-119 138 138 138 255 255 255 F F F
152152024-02-03T17:29:20.631Z 6 120-139 140 140 140 257 257 257 F F F
152162024-02-03T17:29:20.631Z 7 140-159 128 128 128 235 235 235 F F F
152172024-02-03T17:29:20.631Z 8 160-179 137 137 137 252 252 252 F F F
152182024-02-03T17:29:20.631Z 9 180-199 137 137 137 252 252 252 F F F
152192024-02-03T17:29:20.631Z 10 200-219 131 131 131 239 239 239 F F F
152202024-02-03T17:29:20.632Z 11 220-239 138 138 138 254 254 254 F F F
152212024-02-03T17:29:20.632Z 12 240-259 135 135 135 247 247 247 F F F
152222024-02-03T17:29:20.632Z 13 260-279 136 136 136 248 248 248 F F F
152232024-02-03T17:29:20.632Z 14 280-299 138 138 138 253 253 253 F F F
152242024-02-03T17:29:20.632Z 15 300-319 140 140 140 259 259 259 F F F
152252024-02-03T17:29:20.632Z 16 320-339 140 140 140 259 259 259 F F F
152262024-02-03T17:29:20.632Z 17 340-359 138 138 138 255 255 255 F F F
152272024-02-03T17:29:20.632Z 18 360-379 140 140 140 258 258 258 F F F
152282024-02-03T17:29:20.633Z 19 380-399 141 141 141 260 260 260 F F F
152292024-02-03T17:29:20.633Z 20 400-419 141 141 141 260 260 260 F F F
152302024-02-03T17:29:20.633Z 21 420-439 138 138 138 254 254 254 F F F
152312024-02-03T17:29:20.633Z 22 440-459 141 141 141 260 260 260 F F F
152322024-02-03T17:29:20.633Z 23 460-479 135 135 135 247 247 247 F F F
152332024-02-03T17:29:20.633Z 24 480-499 139 139 139 256 256 256 F F F
152342024-02-03T17:29:20.633Z 25 500-519 140 140 140 258 258 258 F F F
152352024-02-03T17:29:20.633Z 26 520-539 140 140 140 259 259 259 F F F
152362024-02-03T17:29:20.633Z 27 540-559 141 141 141 260 260 260 F F F
152372024-02-03T17:29:20.633Z 28 560-579 141 141 141 260 260 260 F F F
152382024-02-03T17:29:20.633Z 29 580-599 141 141 141 260 260 260 F F F
152392024-02-03T17:29:20.633ZMax gen: 141, Max flush: 260
152402024-02-03T17:29:20.633Z
152412024-02-03T17:29:20.634ZBegin loop 71
152422024-02-03T17:29:20.634ZDownstairs to restart: 0
152432024-02-03T17:29:20.660Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.832832307Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703}
152442024-02-03T17:29:20.661Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833376968Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703}
152452024-02-03T17:29:20.661Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
152462024-02-03T17:29:20.665Z2024-02-03T17:29:19.833422691Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703}
152472024-02-03T17:29:20.665Z{"msg":"Crucible bf4ae442-0661-49b9-8445-39ac1658afb6 has session id: 90eebcf6-7651-4629-88e3-7068be95bbe6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833592862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152482024-02-03T17:29:20.665Z{"msg":"Upstairs opts: Upstairs UUID: bf4ae442-0661-49b9-8445-39ac1658afb6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833662055Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152492024-02-03T17:29:20.665Z{"msg":"Crucible stats registered with UUID: bf4ae442-0661-49b9-8445-39ac1658afb6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833726696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152502024-02-03T17:29:20.665Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833768912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703}
152512024-02-03T17:29:20.665Z{"msg":"bf4ae442-0661-49b9-8445-39ac1658afb6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833822333Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152522024-02-03T17:29:20.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833862958Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152532024-02-03T17:29:20.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833921403Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152542024-02-03T17:29:20.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.833964485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152552024-02-03T17:29:20.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.834027527Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152562024-02-03T17:29:20.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.834116912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152572024-02-03T17:29:20.665Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.834234401Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152582024-02-03T17:29:20.665Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.834277611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152592024-02-03T17:29:20.665Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.834414286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152602024-02-03T17:29:20.665Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.834532894Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152612024-02-03T17:29:20.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.834660094Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152622024-02-03T17:29:20.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.83470649Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152632024-02-03T17:29:20.666Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.83476819Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152642024-02-03T17:29:20.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.835936195Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152652024-02-03T17:29:20.666Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.836023324Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152662024-02-03T17:29:20.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.836069297Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152672024-02-03T17:29:20.666Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.836144788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152682024-02-03T17:29:20.666Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.836749823Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152692024-02-03T17:29:20.666Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.836872319Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152702024-02-03T17:29:20.671Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.837061788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152712024-02-03T17:29:20.671Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.837133191Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152722024-02-03T17:29:20.671Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.837685816Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152732024-02-03T17:29:20.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.838178707Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152742024-02-03T17:29:20.671Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.838267007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152752024-02-03T17:29:20.671Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.83831408Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152762024-02-03T17:29:20.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.83835926Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152772024-02-03T17:29:20.671Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.838400418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152782024-02-03T17:29:20.671Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.838455936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152792024-02-03T17:29:20.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.838720366Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152802024-02-03T17:29:20.671Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.838803272Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152812024-02-03T17:29:20.671Z{"msg":"[0]R flush_numbers[0..12]: [257, 260, 260, 251, 257, 255, 257, 235, 252, 252, 239, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.838876541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152822024-02-03T17:29:20.671Z{"msg":"[0]R generation[0..12]: [140, 141, 141, 136, 140, 138, 140, 128, 137, 137, 131, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.838950458Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152832024-02-03T17:29:20.671Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839021896Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152842024-02-03T17:29:20.671Z{"msg":"[1]R flush_numbers[0..12]: [257, 260, 260, 251, 257, 255, 257, 235, 252, 252, 239, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839095662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152852024-02-03T17:29:20.672Z{"msg":"[1]R generation[0..12]: [140, 141, 141, 136, 140, 138, 140, 128, 137, 137, 131, 138]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
152862024-02-03T17:29:20.672Z,"time":"2024-02-03T17:29:19.839165792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152872024-02-03T17:29:20.672Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839244664Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152882024-02-03T17:29:20.672Z{"msg":"[2]R flush_numbers[0..12]: [257, 260, 260, 251, 257, 255, 257, 235, 252, 252, 239, 254]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839318416Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152892024-02-03T17:29:20.672Z{"msg":"[2]R generation[0..12]: [140, 141, 141, 136, 140, 138, 140, 128, 137, 137, 131, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839392936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152902024-02-03T17:29:20.672Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.8394672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152912024-02-03T17:29:20.672Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839538784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152922024-02-03T17:29:20.672Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839608042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152932024-02-03T17:29:20.672Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.83967774Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152942024-02-03T17:29:20.672Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839747362Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152952024-02-03T17:29:20.672Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839817913Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152962024-02-03T17:29:20.672Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839884287Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152972024-02-03T17:29:20.672Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839950246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152982024-02-03T17:29:20.672Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.839978155Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
152992024-02-03T17:29:20.672ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153002024-02-03T17:29:20.672Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.840024391Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
153012024-02-03T17:29:20.672Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.840047201Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
153022024-02-03T17:29:20.672Z{"msg":"bf4ae442-0661-49b9-8445-39ac1658afb6 is now active with session: 90eebcf6-7651-4629-88e3-7068be95bbe6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.840087806Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
153032024-02-03T17:29:20.672Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.840109728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703,"session_id":"90eebcf6-7651-4629-88e3-7068be95bbe6"}
153042024-02-03T17:29:20.672Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:19.840149397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703}
153052024-02-03T17:29:20.672ZLoading write count information from file "/tmp/test_repair_verify.data"
153062024-02-03T17:29:20.672ZRun Repair workload
153072024-02-03T17:29:20.673Z01/30 Write block 506 len 12288 data: 23 22 18
153082024-02-03T17:29:20.677Z02/30 Read block 218 len 16384
153092024-02-03T17:29:20.677Z03/30 Write block 413 len 4096 data: 11
153102024-02-03T17:29:20.677Z04/30 Write block 22 len 36864 data: 8 8 12 12 10 9 11 9 6
153112024-02-03T17:29:20.677Z05/30 Write block 14 len 20480 data: 11 10 10 9 8
153122024-02-03T17:29:20.677Z06/30 Write block 253 len 20480 data: 9 11 11 10 7
153132024-02-03T17:29:20.681Z07/30 Read block 424 len 24576
153142024-02-03T17:29:20.681Z08/30 Flush
153152024-02-03T17:29:20.684Z09/30 Write block 31 len 4096 data: 8
153162024-02-03T17:29:20.684Z10/30 Read block 584 len 12288
153172024-02-03T17:29:20.684Z11/30 Read block 394 len 24576
153182024-02-03T17:29:20.687Z12/30 Read block 549 len 36864
153192024-02-03T17:29:20.687Z13/30 Read block 377 len 40960
153202024-02-03T17:29:20.691Z14/30 Write block 234 len 36864 data: 12 13 12 13 10 12 9 9 6
153212024-02-03T17:29:20.691Z15/30 Read block 466 len 20480
153222024-02-03T17:29:20.694Z16/30 Read block 244 len 4096
153232024-02-03T17:29:20.695Z17/30 Write block 570 len 8192 data: 10 9
153242024-02-03T17:29:20.695Z18/30 Write block 237 len 4096 data: 14
153252024-02-03T17:29:20.698Z19/30 Write block 431 len 36864 data: 12 11 9 11 11 11 12 13 14
153262024-02-03T17:29:20.698Z20/30 Read block 185 len 12288
153272024-02-03T17:29:20.701Z21/30 Write block 228 len 20480 data: 16 11 8 11 12
153282024-02-03T17:29:20.701Z22/30 Read block 274 len 20480
153292024-02-03T17:29:20.701Z23/30 Write block 409 len 20480 data: 11 10 10 9 12
153302024-02-03T17:29:20.705Z24/30 Read block 197 len 28672
153312024-02-03T17:29:20.705Z25/30 Flush
153322024-02-03T17:29:20.711Z26/30 Write block 115 len 8192 data: 15 16
153332024-02-03T17:29:20.711Z27/30 Read block 102 len 28672
153342024-02-03T17:29:20.715Z28/30 Read block 103 len 4096
153352024-02-03T17:29:20.715Z29/30 Write block 460 len 32768 data: 13 11 13 13 12 13 10 11
153362024-02-03T17:29:20.715Z30/30 Read block 239 len 20480
153372024-02-03T17:29:20.715Z----------------------------------------------------------------
153382024-02-03T17:29:20.715Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
153392024-02-03T17:29:20.715ZStates: Active Active Active
153402024-02-03T17:29:20.715ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153412024-02-03T17:29:20.715Z 1 Acked 1000 Write 3 Done Done Done false
153422024-02-03T17:29:20.715Z 2 Acked 1001 Read 4 Done Done Done false
153432024-02-03T17:29:20.715Z 3 Acked 1002 Write 1 Sent Done Done false
153442024-02-03T17:29:20.715Z 4 Acked 1003 Write 9 Sent Done Done false
153452024-02-03T17:29:20.715Z 5 Acked 1004 Write 5 Sent Done Done false
153462024-02-03T17:29:20.716Z 6 Acked 1005 Write 5 Sent Done Done false
153472024-02-03T17:29:20.716Z 7 Acked 1006 Read 6 Sent Done Done false
153482024-02-03T17:29:20.716Z 8 Acked 1007 Flush 0 Sent Done Done false
153492024-02-03T17:29:20.716Z 9 Acked 1008 Write 1 Sent Done Done false
153502024-02-03T17:29:20.716Z 10 Acked 1009 Read 3 Sent Done Done false
153512024-02-03T17:29:20.716Z 11 Acked 1010 Read 6 Sent Done Done false
153522024-02-03T17:29:20.716Z 12 Acked 1011 Read 9 Sent Done Done false
153532024-02-03T17:29:20.716Z 13 Acked 1012 Read 10 Sent Done Done false
153542024-02-03T17:29:20.716Z 14 Acked 1013 Write 9 Sent Done Done false
153552024-02-03T17:29:20.716Z 15 Acked 1014 Read 5 Sent Done Done false
153562024-02-03T17:29:20.716Z 16 Acked 1015 Read 1 Sent Done Done false
153572024-02-03T17:29:20.716Z 17 Acked 1016 Write 2 Sent Done Done false
153582024-02-03T17:29:20.716Z 18 Acked 1017 Write 1 Sent Done Done false
153592024-02-03T17:29:20.716Z 19 Acked 1018 Write 9 Sent Done Done false
153602024-02-03T17:29:20.716Z 20 Acked 1019 Read 3 Sent Done Done false
153612024-02-03T17:29:20.716Z 21 Acked 1020 Write 5 Sent Done Done false
153622024-02-03T17:29:20.716Z 22 Acked 1021 Read 5 Sent Done Done false
153632024-02-03T17:29:20.716Z 23 Acked 1022 Write 5 Sent Done Done false
153642024-02-03T17:29:20.716Z 24 Acked 1023 Read 7 Sent Done Done false
153652024-02-03T17:29:20.716Z 25 Acked 1024 Flush 0 Sent Done Done false
153662024-02-03T17:29:20.716Z 26 Acked 1025 Write 2 Sent Done Done false
153672024-02-03T17:29:20.716Z 27 Acked 1026 Read 7 Sent Done Done false
153682024-02-03T17:29:20.716Z 28 Acked 1027 Read 1 Sent Done Done false
153692024-02-03T17:29:20.717Z 29 Acked 1028 Write 8 Sent Done Done false
153702024-02-03T17:29:20.717Z 30 Acked 1029 Read 5 Sent Done Done false
153712024-02-03T17:29:20.717Z STATES DS:0 DS:1 DS:2 TOTAL
153722024-02-03T17:29:20.717Z New 0 0 0 0
153732024-02-03T17:29:20.717Z Sent 28 0 0 28
153742024-02-03T17:29:20.717Z Done 2 30 30 62
153752024-02-03T17:29:20.717Z Skipped 0 0 0 0
153762024-02-03T17:29:20.717Z Error 0 0 0 0
153772024-02-03T17:29:20.717ZLast Flush: 0 1024 1024
153782024-02-03T17:29:20.717ZDownstairs last five completed:
153792024-02-03T17:29:20.717ZUpstairs last five completed: 30 29 28 27 26
153802024-02-03T17:29:20.720Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:19.887457177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1703}
153812024-02-03T17:29:20.720ZWrote out file "/tmp/test_repair_verify.data"
153822024-02-03T17:29:20.724Z
153832024-02-03T17:29:20.730ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153842024-02-03T17:29:20.756ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153852024-02-03T17:29:20.756Z 0 000-019 140 142 142 257 261 261 F F F <---
153862024-02-03T17:29:20.760Z 1 020-039 141 142 142 260 262 262 F F F <---
153872024-02-03T17:29:20.760Z 2 040-059 141 141 141 260 260 260 F F F
153882024-02-03T17:29:20.760Z 3 060-079 136 136 136 251 251 251 F F F
153892024-02-03T17:29:20.760Z 4 080-099 140 140 140 257 257 257 F F F
153902024-02-03T17:29:20.760Z 5 100-119 138 138 138 255 255 255 F T T <---
153912024-02-03T17:29:20.761Z 6 120-139 140 140 140 257 257 257 F F F
153922024-02-03T17:29:20.761Z 7 140-159 128 128 128 235 235 235 F F F
153932024-02-03T17:29:20.761Z 8 160-179 137 137 137 252 252 252 F F F
153942024-02-03T17:29:20.761Z 9 180-199 137 137 137 252 252 252 F F F
153952024-02-03T17:29:20.761Z 10 200-219 131 131 131 239 239 239 F F F
153962024-02-03T17:29:20.761Z 11 220-239 138 142 142 254 262 262 F F F <---
153972024-02-03T17:29:20.761Z 12 240-259 135 142 142 247 262 262 F F F <---
153982024-02-03T17:29:20.761Z 13 260-279 136 136 136 248 248 248 F F F
153992024-02-03T17:29:20.761Z 14 280-299 138 138 138 253 253 253 F F F
154002024-02-03T17:29:20.761Z 15 300-319 140 140 140 259 259 259 F F F
154012024-02-03T17:29:20.761Z 16 320-339 140 140 140 259 259 259 F F F
154022024-02-03T17:29:20.761Z 17 340-359 138 138 138 255 255 255 F F F
154032024-02-03T17:29:20.762Z 18 360-379 140 140 140 258 258 258 F F F
154042024-02-03T17:29:20.762Z 19 380-399 141 141 141 260 260 260 F F F
154052024-02-03T17:29:20.762Z 20 400-419 141 142 142 260 262 262 F F F <---
154062024-02-03T17:29:20.762Z 21 420-439 138 142 142 254 262 262 F F F <---
154072024-02-03T17:29:20.762Z 22 440-459 141 141 141 260 260 260 F F F
154082024-02-03T17:29:20.762Z 23 460-479 135 135 135 247 247 247 F T T <---
154092024-02-03T17:29:20.762Z 24 480-499 139 139 139 256 256 256 F F F
154102024-02-03T17:29:20.762Z 25 500-519 140 142 142 258 261 261 T F F <---
154112024-02-03T17:29:20.762Z 26 520-539 140 140 140 259 259 259 F F F
154122024-02-03T17:29:20.762Z 27 540-559 141 141 141 260 260 260 F F F
154132024-02-03T17:29:20.762Z 28 560-579 141 142 142 260 262 262 F F F <---
154142024-02-03T17:29:20.763Z 29 580-599 141 141 141 260 260 260 F F F
154152024-02-03T17:29:20.763ZMax gen: 142, Max flush: 262
154162024-02-03T17:29:20.763ZError: Difference in extent metadata found!
154172024-02-03T17:29:20.767ZA Difference in extent metadata is expected here
154182024-02-03T17:29:20.767ZOn loop 71
154192024-02-03T17:29:20.767Z
154202024-02-03T17:29:20.767ZVerifying data now
154212024-02-03T17:29:25.670ZNo change after verify
154222024-02-03T17:29:25.670ZLoop: 71 Downstairs dump after verify (and repair):
154232024-02-03T17:29:25.693ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154242024-02-03T17:29:25.693Z 0 000-019 142 142 142 261 261 261 F F F
154252024-02-03T17:29:25.700Z 1 020-039 142 142 142 262 262 262 F F F
154262024-02-03T17:29:25.700Z 2 040-059 141 141 141 260 260 260 F F F
154272024-02-03T17:29:25.700Z 3 060-079 136 136 136 251 251 251 F F F
154282024-02-03T17:29:25.700Z 4 080-099 140 140 140 257 257 257 F F F
154292024-02-03T17:29:25.700Z 5 100-119 143 143 143 263 263 263 F F F
154302024-02-03T17:29:25.700Z 6 120-139 140 140 140 257 257 257 F F F
154312024-02-03T17:29:25.700Z 7 140-159 128 128 128 235 235 235 F F F
154322024-02-03T17:29:25.700Z 8 160-179 137 137 137 252 252 252 F F F
154332024-02-03T17:29:25.700Z 9 180-199 137 137 137 252 252 252 F F F
154342024-02-03T17:29:25.700Z 10 200-219 131 131 131 239 239 239 F F F
154352024-02-03T17:29:25.700Z 11 220-239 142 142 142 262 262 262 F F F
154362024-02-03T17:29:25.701Z 12 240-259 142 142 142 262 262 262 F F F
154372024-02-03T17:29:25.701Z 13 260-279 136 136 136 248 248 248 F F F
154382024-02-03T17:29:25.701Z 14 280-299 138 138 138 253 253 253 F F F
154392024-02-03T17:29:25.701Z 15 300-319 140 140 140 259 259 259 F F F
154402024-02-03T17:29:25.701Z 16 320-339 140 140 140 259 259 259 F F F
154412024-02-03T17:29:25.701Z 17 340-359 138 138 138 255 255 255 F F F
154422024-02-03T17:29:25.701Z 18 360-379 140 140 140 258 258 258 F F F
154432024-02-03T17:29:25.701Z 19 380-399 141 141 141 260 260 260 F F F
154442024-02-03T17:29:25.701Z 20 400-419 142 142 142 262 262 262 F F F
154452024-02-03T17:29:25.701Z 21 420-439 142 142 142 262 262 262 F F F
154462024-02-03T17:29:25.701Z 22 440-459 141 141 141 260 260 260 F F F
154472024-02-03T17:29:25.701Z 23 460-479 143 143 143 263 263 263 F F F
154482024-02-03T17:29:25.701Z 24 480-499 139 139 139 256 256 256 F F F
154492024-02-03T17:29:25.701Z 25 500-519 142 142 142 261 261 261 F F F
154502024-02-03T17:29:25.701Z 26 520-539 140 140 140 259 259 259 F F F
154512024-02-03T17:29:25.701Z 27 540-559 141 141 141 260 260 260 F F F
154522024-02-03T17:29:25.701Z 28 560-579 142 142 142 262 262 262 F F F
154532024-02-03T17:29:25.701Z 29 580-599 141 141 141 260 260 260 F F F
154542024-02-03T17:29:25.701ZMax gen: 143, Max flush: 263
154552024-02-03T17:29:25.701Z
154562024-02-03T17:29:25.701ZBegin loop 72
154572024-02-03T17:29:25.701ZDownstairs to restart: 1
154582024-02-03T17:29:25.730Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.903121035Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711}
154592024-02-03T17:29:25.730Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.903668861Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711}
154602024-02-03T17:29:25.734Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.903721018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711}
154612024-02-03T17:29:25.735Z{"msg":"Crucible 676f9f4b-e5cc-4bfb-b3e5-253fc0438803 has session id: 5cb73086-0cd4-40fd-8ec9-40d4f0c370d7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.903793609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154622024-02-03T17:29:25.735Z{"msg":"Upstairs opts: Upstairs UUID: 676f9f4b-e5cc-4bfb-b3e5-253fc0438803, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.903908068Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154632024-02-03T17:29:25.735Z{"msg":"Crucible stats registered with UUID: 676f9f4b-e5cc-4bfb-b3e5-253fc0438803","v":0,"name":"crucible","level":30Crucible runtime is spawned
154642024-02-03T17:29:25.735Z,"time":"2024-02-03T17:29:24.903957868Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154652024-02-03T17:29:25.735Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904129161Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711}
154662024-02-03T17:29:25.735Z{"msg":"676f9f4b-e5cc-4bfb-b3e5-253fc0438803 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904180094Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154672024-02-03T17:29:25.735Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904204541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154682024-02-03T17:29:25.735Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904261237Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154692024-02-03T17:29:25.735Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904330211Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154702024-02-03T17:29:25.735Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904382283Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154712024-02-03T17:29:25.735Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904425466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154722024-02-03T17:29:25.735Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904492285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154732024-02-03T17:29:25.735Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904518824Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154742024-02-03T17:29:25.735Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904555437Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154752024-02-03T17:29:25.735Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904598254Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154762024-02-03T17:29:25.735Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904748348Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154772024-02-03T17:29:25.735Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.904995595Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154782024-02-03T17:29:25.735Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.905127386Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154792024-02-03T17:29:25.735Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.906103863Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154802024-02-03T17:29:25.735Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.906200606Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154812024-02-03T17:29:25.735Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.906316819Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154822024-02-03T17:29:25.735Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.906396516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154832024-02-03T17:29:25.736Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.907054255Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154842024-02-03T17:29:25.736Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.907139136Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154852024-02-03T17:29:25.736Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.907166732Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154862024-02-03T17:29:25.740Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.907349449Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154872024-02-03T17:29:25.740Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908245412Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154882024-02-03T17:29:25.740Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908336169Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154892024-02-03T17:29:25.740Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908386512Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154902024-02-03T17:29:25.740Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908452264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154912024-02-03T17:29:25.740Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908493622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154922024-02-03T17:29:25.740Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908548758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154932024-02-03T17:29:25.740Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908592173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154942024-02-03T17:29:25.740Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908911038Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154952024-02-03T17:29:25.740Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.908965694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154962024-02-03T17:29:25.740Z{"msg":"[0]R flush_numbers[0..12]: [261, 262, 260, 251, 257, 263, 257, 235, 252, 252, 239, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909039802Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154972024-02-03T17:29:25.740Z{"msg":"[0]R generation[0..12]: [142, 142, 141, 136, 140, 143, 140, 128, 137, 137, 131, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909111462Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154982024-02-03T17:29:25.740Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909145335Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
154992024-02-03T17:29:25.740Z{"msg":"[1]R flush_numbers[0..12]: [261, 262, 260, 251, 257, 263, 257, 235, 252, 252, 239, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.9091908Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155002024-02-03T17:29:25.740Z{"msg":"[1]R generation[0..12]: [142, 142, 141, 136, 140, 143, 140, 128, 137, 137, 131, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909216164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155012024-02-03T17:29:25.740Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
155022024-02-03T17:29:25.741Z,"time":"2024-02-03T17:29:24.909259478Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155032024-02-03T17:29:25.741Z{"msg":"[2]R flush_numbers[0..12]: [261, 262, 260, 251, 257, 263, 257, 235, 252, 252, 239, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909288477Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155042024-02-03T17:29:25.741Z{"msg":"[2]R generation[0..12]: [142, 142, 141, 136, 140, 143, 140, 128, 137, 137, 131, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909331452Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155052024-02-03T17:29:25.741Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909357003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155062024-02-03T17:29:25.741Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909397738Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155072024-02-03T17:29:25.741Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909422036Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155082024-02-03T17:29:25.741Z{"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909443808Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155092024-02-03T17:29:25.741Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909465145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155102024-02-03T17:29:25.741Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909513478Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155112024-02-03T17:29:25.741Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909535719Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155122024-02-03T17:29:25.741Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909558166Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155132024-02-03T17:29:25.741Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909604996Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155142024-02-03T17:29:25.741Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909629249Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155152024-02-03T17:29:25.741Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909669Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155162024-02-03T17:29:25.741Z{"msg":"676f9f4b-e5cc-4bfb-b3e5-253fc0438803 is now active with session: 5cb73086-0cd4-40fd-8ec9-40d4f0c370d7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909693361Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155172024-02-03T17:29:25.741Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909712876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711,"session_id":"5cb73086-0cd4-40fd-8ec9-40d4f0c370d7"}
155182024-02-03T17:29:25.741Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:24.909752637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711}
155192024-02-03T17:29:25.741ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155202024-02-03T17:29:25.741ZLoading write count information from file "/tmp/test_repair_verify.data"
155212024-02-03T17:29:25.741ZRun Repair workload
155222024-02-03T17:29:25.741Z01/30 Write block 588 len 8192 data: 11 14
155232024-02-03T17:29:25.745Z02/30 Write block 257 len 12288 data: 8 8 7
155242024-02-03T17:29:25.745Z03/30 Flush
155252024-02-03T17:29:25.745Z04/30 Write block 296 len 20480 data: 11 12 8 5 8
155262024-02-03T17:29:25.745Z05/30 Read block 191 len 12288
155272024-02-03T17:29:25.748Z06/30 Write block 176 len 8192 data: 10 13
155282024-02-03T17:29:25.748Z07/30 Write block 274 len 20480 data: 16 18 18 17 17
155292024-02-03T17:29:25.751Z08/30 Read block 329 len 32768
155302024-02-03T17:29:25.751Z09/30 Write block 274 len 16384 data: 17 19 19 18
155312024-02-03T17:29:25.757Z10/30 Write block 548 len 24576 data: 14 15 13 12 10 10
155322024-02-03T17:29:25.757Z11/30 Read block 169 len 16384
155332024-02-03T17:29:25.760Z12/30 Write block 148 len 36864 data: 5 6 11 12 8 9 10 12 12
155342024-02-03T17:29:25.760Z13/30 Read block 454 len 32768
155352024-02-03T17:29:25.760Z14/30 Flush
155362024-02-03T17:29:25.763Z15/30 Write block 552 len 40960 data: 11 11 10 10 11 10 11 12 8 6
155372024-02-03T17:29:25.763Z16/30 Read block 319 len 12288
155382024-02-03T17:29:25.766Z17/30 Flush
155392024-02-03T17:29:25.769Z18/30 Write block 70 len 16384 data: 7 7 6 8
155402024-02-03T17:29:25.769Z19/30 Read block 557 len 12288
155412024-02-03T17:29:25.772Z20/30 Write block 157 len 8192 data: 13 12
155422024-02-03T17:29:25.772Z21/30 Write block 529 len 4096 data: 13
155432024-02-03T17:29:25.772Z22/30 Write block 583 len 24576 data: 13 14 11 15 12 12
155442024-02-03T17:29:25.773Z23/30 Flush
155452024-02-03T17:29:25.776Z24/30 Flush
155462024-02-03T17:29:25.779Z25/30 Flush
155472024-02-03T17:29:25.779Z26/30 Write block 396 len 32768 data: 12 9 8 9 10 8 8 11
155482024-02-03T17:29:25.782Z27/30 Write block 302 len 36864 data: 9 9 9 12 13 10 12 12 10
155492024-02-03T17:29:25.782Z28/30 Write block 521 len 4096 data: 6
155502024-02-03T17:29:25.782Z29/30 Write block 95 len 28672 data: 9 10 11 8 10 11 11
155512024-02-03T17:29:25.786Z30/30 Write block 582 len 20480 data: 16 14 15 12 16
155522024-02-03T17:29:25.786Z----------------------------------------------------------------
155532024-02-03T17:29:25.786Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
155542024-02-03T17:29:25.786ZStates: Active Active Active
155552024-02-03T17:29:25.787ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155562024-02-03T17:29:25.787Z 1 Acked 1000 Write 2 Done Sent Done false
155572024-02-03T17:29:25.787Z 2 Acked 1001 Write 3 Done Sent Done false
155582024-02-03T17:29:25.787Z 3 Acked 1002 Flush 0 Done Sent Done false
155592024-02-03T17:29:25.787Z 4 Acked 1003 Write 5 Done Sent Done false
155602024-02-03T17:29:25.787Z 5 Acked 1004 Read 3 Done Sent Done false
155612024-02-03T17:29:25.787Z 6 Acked 1005 Write 2 Done Sent Done false
155622024-02-03T17:29:25.787Z 7 Acked 1006 Write 5 Done Sent Done false
155632024-02-03T17:29:25.787Z 8 Acked 1007 Read 8 Done Sent Done false
155642024-02-03T17:29:25.787Z 9 Acked 1008 Write 4 Done Sent Done false
155652024-02-03T17:29:25.787Z 10 Acked 1009 Write 6 Done Sent Done false
155662024-02-03T17:29:25.787Z 11 Acked 1010 Read 4 Done Sent Done false
155672024-02-03T17:29:25.787Z 12 Acked 1011 Write 9 Done Sent Done false
155682024-02-03T17:29:25.787Z 13 Acked 1012 Read 8 Done Sent Done false
155692024-02-03T17:29:25.787Z 14 Acked 1013 Flush 0 Done Sent Done false
155702024-02-03T17:29:25.787Z 15 Acked 1014 Write 10 Done Sent Done false
155712024-02-03T17:29:25.787Z 16 Acked 1015 Read 3 Done Sent Done false
155722024-02-03T17:29:25.787Z 17 Acked 1016 Flush 0 Done Sent Done false
155732024-02-03T17:29:25.787Z 18 Acked 1017 Write 4 Done Sent Done false
155742024-02-03T17:29:25.787Z 19 Acked 1018 Read 3 Done Sent Done false
155752024-02-03T17:29:25.787Z 20 Acked 1019 Write 2 Done Sent Done false
155762024-02-03T17:29:25.787Z 21 Acked 1020 Write 1 Done Sent Done false
155772024-02-03T17:29:25.787Z 22 Acked 1021 Write 6 Done Sent Done false
155782024-02-03T17:29:25.788Z 23 Acked 1022 Flush 0 Done Sent Done false
155792024-02-03T17:29:25.788Z 24 Acked 1023 Flush 0 Done Sent Done false
155802024-02-03T17:29:25.788Z 25 Acked 1024 Flush 0 Done Sent Done false
155812024-02-03T17:29:25.788Z 26 Acked 1025 Write 8 Done Sent Done false
155822024-02-03T17:29:25.788Z 27 Acked 1026 Write 9 Done Sent Done false
155832024-02-03T17:29:25.788Z 28 Acked 1027 Write 1 Done Sent Done false
155842024-02-03T17:29:25.788Z 29 Acked 1028 Write 7 Done Sent Done false
155852024-02-03T17:29:25.788Z 30 Acked 1029 Write 5 Sent Sent Sent false
155862024-02-03T17:29:25.788Z STATES DS:0 DS:1 DS:2 TOTAL
155872024-02-03T17:29:25.788Z New 0 0 0 0
155882024-02-03T17:29:25.788Z Sent 1 30 1 32
155892024-02-03T17:29:25.788Z Done 29 0 29 58
155902024-02-03T17:29:25.788Z Skipped 0 0 0 0
155912024-02-03T17:29:25.788Z Error 0 0 0 0
155922024-02-03T17:29:25.788ZLast Flush: 1024 0 1024
155932024-02-03T17:29:25.788ZDownstairs last five completed:
155942024-02-03T17:29:25.788ZUpstairs last five completed: 30 29 28 27 26
155952024-02-03T17:29:25.788Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:24.957403801Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1711}
155962024-02-03T17:29:25.788ZWrote out file "/tmp/test_repair_verify.data"
155972024-02-03T17:29:25.792Z
155982024-02-03T17:29:25.800ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155992024-02-03T17:29:25.832ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156002024-02-03T17:29:25.832Z 0 000-019 142 142 142 261 261 261 F F F
156012024-02-03T17:29:25.840Z 1 020-039 142 142 142 262 262 262 F F F
156022024-02-03T17:29:25.841Z 2 040-059 141 141 141 260 260 260 F F F
156032024-02-03T17:29:25.841Z 3 060-079 144 136 144 267 251 267 F F F <---
156042024-02-03T17:29:25.841Z 4 080-099 140 140 140 257 257 257 T F T <---
156052024-02-03T17:29:25.841Z 5 100-119 143 143 143 263 263 263 T F T <---
156062024-02-03T17:29:25.841Z 6 120-139 140 140 140 257 257 257 F F F
156072024-02-03T17:29:25.841Z 7 140-159 144 128 144 267 235 267 F F F <---
156082024-02-03T17:29:25.841Z 8 160-179 144 137 144 265 252 265 F F F <---
156092024-02-03T17:29:25.841Z 9 180-199 137 137 137 252 252 252 F F F
156102024-02-03T17:29:25.842Z 10 200-219 131 131 131 239 239 239 F F F
156112024-02-03T17:29:25.842Z 11 220-239 142 142 142 262 262 262 F F F
156122024-02-03T17:29:25.842Z 12 240-259 144 142 144 264 262 264 F F F <---
156132024-02-03T17:29:25.842Z 13 260-279 144 136 144 265 248 265 F F F <---
156142024-02-03T17:29:25.842Z 14 280-299 144 138 144 265 253 265 F F F <---
156152024-02-03T17:29:25.842Z 15 300-319 144 140 144 265 259 265 T F T <---
156162024-02-03T17:29:25.842Z 16 320-339 140 140 140 259 259 259 F F F
156172024-02-03T17:29:25.843Z 17 340-359 138 138 138 255 255 255 F F F
156182024-02-03T17:29:25.843Z 18 360-379 140 140 140 258 258 258 F F F
156192024-02-03T17:29:25.843Z 19 380-399 141 141 141 260 260 260 T F T <---
156202024-02-03T17:29:25.843Z 20 400-419 142 142 142 262 262 262 T F T <---
156212024-02-03T17:29:25.843Z 21 420-439 142 142 142 262 262 262 F F F
156222024-02-03T17:29:25.843Z 22 440-459 141 141 141 260 260 260 F F F
156232024-02-03T17:29:25.843Z 23 460-479 143 143 143 263 263 263 F F F
156242024-02-03T17:29:25.843Z 24 480-499 139 139 139 256 256 256 F F F
156252024-02-03T17:29:25.843Z 25 500-519 142 142 142 261 261 261 F F F
156262024-02-03T17:29:25.843Z 26 520-539 144 140 144 267 259 267 T F T <---
156272024-02-03T17:29:25.844Z 27 540-559 144 141 144 266 260 266 F F F <---
156282024-02-03T17:29:25.844Z 28 560-579 144 142 144 266 262 266 F F F <---
156292024-02-03T17:29:25.844Z 29 580-599 144 141 144 267 260 267 T F T <---
156302024-02-03T17:29:25.844ZMax gen: 144, Max flush: 267
156312024-02-03T17:29:25.844ZError: Difference in extent metadata found!
156322024-02-03T17:29:25.844ZA Difference in extent metadata is expected here
156332024-02-03T17:29:25.844ZOn loop 72
156342024-02-03T17:29:25.844Z
156352024-02-03T17:29:25.845ZVerifying data now
156362024-02-03T17:29:31.150ZNo change after verify
156372024-02-03T17:29:31.151ZLoop: 72 Downstairs dump after verify (and repair):
156382024-02-03T17:29:31.171ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156392024-02-03T17:29:31.171Z 0 000-019 142 142 142 261 261 261 F F F
156402024-02-03T17:29:31.171Z 1 020-039 142 142 142 262 262 262 F F F
156412024-02-03T17:29:31.171Z 2 040-059 141 141 141 260 260 260 F F F
156422024-02-03T17:29:31.172Z 3 060-079 144 144 144 267 267 267 F F F
156432024-02-03T17:29:31.177Z 4 080-099 145 145 145 268 268 268 F F F
156442024-02-03T17:29:31.177Z 5 100-119 145 145 145 268 268 268 F F F
156452024-02-03T17:29:31.178Z 6 120-139 140 140 140 257 257 257 F F F
156462024-02-03T17:29:31.178Z 7 140-159 144 144 144 267 267 267 F F F
156472024-02-03T17:29:31.178Z 8 160-179 144 144 144 265 265 265 F F F
156482024-02-03T17:29:31.178Z 9 180-199 137 137 137 252 252 252 F F F
156492024-02-03T17:29:31.178Z 10 200-219 131 131 131 239 239 239 F F F
156502024-02-03T17:29:31.178Z 11 220-239 142 142 142 262 262 262 F F F
156512024-02-03T17:29:31.178Z 12 240-259 144 144 144 264 264 264 F F F
156522024-02-03T17:29:31.179Z 13 260-279 144 144 144 265 265 265 F F F
156532024-02-03T17:29:31.179Z 14 280-299 144 144 144 265 265 265 F F F
156542024-02-03T17:29:31.179Z 15 300-319 145 145 145 268 268 268 F F F
156552024-02-03T17:29:31.179Z 16 320-339 140 140 140 259 259 259 F F F
156562024-02-03T17:29:31.179Z 17 340-359 138 138 138 255 255 255 F F F
156572024-02-03T17:29:31.179Z 18 360-379 140 140 140 258 258 258 F F F
156582024-02-03T17:29:31.179Z 19 380-399 145 145 145 268 268 268 F F F
156592024-02-03T17:29:31.179Z 20 400-419 145 145 145 268 268 268 F F F
156602024-02-03T17:29:31.179Z 21 420-439 142 142 142 262 262 262 F F F
156612024-02-03T17:29:31.179Z 22 440-459 141 141 141 260 260 260 F F F
156622024-02-03T17:29:31.179Z 23 460-479 143 143 143 263 263 263 F F F
156632024-02-03T17:29:31.179Z 24 480-499 139 139 139 256 256 256 F F F
156642024-02-03T17:29:31.180Z 25 500-519 142 142 142 261 261 261 F F F
156652024-02-03T17:29:31.180Z 26 520-539 145 145 145 268 268 268 F F F
156662024-02-03T17:29:31.180Z 27 540-559 144 144 144 266 266 266 F F F
156672024-02-03T17:29:31.180Z 28 560-579 144 144 144 266 266 266 F F F
156682024-02-03T17:29:31.180Z 29 580-599 145 145 145 268 268 268 F F F
156692024-02-03T17:29:31.180ZMax gen: 145, Max flush: 268
156702024-02-03T17:29:31.180Z
156712024-02-03T17:29:31.180ZBegin loop 73
156722024-02-03T17:29:31.180ZDownstairs to restart: 2
156732024-02-03T17:29:31.212Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.382660486Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719}
156742024-02-03T17:29:31.212Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383165111Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719}
156752024-02-03T17:29:31.212Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383198758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719}
156762024-02-03T17:29:31.212Z{"msg":"Crucible 92c7a36b-3cf5-4a23-b4e2-2036e80e3348 has session id: c8131774-9b8d-4b96-b175-544c5d8a7ece","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383235925Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156772024-02-03T17:29:31.221Z{"msg":"Upstairs opts: Upstairs UUID: 92c7a36b-3cf5-4a23-b4e2-2036e80e3348, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383298189Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156782024-02-03T17:29:31.222Z{"msg":"Crucible runtime is spawned
156792024-02-03T17:29:31.222ZCrucible stats registered with UUID: 92c7a36b-3cf5-4a23-b4e2-2036e80e3348","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383399433Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156802024-02-03T17:29:31.222Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.38353261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719}
156812024-02-03T17:29:31.222Z{"msg":"92c7a36b-3cf5-4a23-b4e2-2036e80e3348 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383645787Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156822024-02-03T17:29:31.222Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383719185Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156832024-02-03T17:29:31.222Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383753528Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156842024-02-03T17:29:31.222Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383815279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156852024-02-03T17:29:31.222Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383842624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156862024-02-03T17:29:31.222Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383918442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156872024-02-03T17:29:31.223Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.383947288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156882024-02-03T17:29:31.223Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.384095493Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156892024-02-03T17:29:31.223Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.384202525Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156902024-02-03T17:29:31.223Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.3842814Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156912024-02-03T17:29:31.223Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.384414447Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156922024-02-03T17:29:31.223Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.384457658Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156932024-02-03T17:29:31.223Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.38448232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156942024-02-03T17:29:31.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.385583912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156952024-02-03T17:29:31.223Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.385728003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156962024-02-03T17:29:31.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.385782381Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156972024-02-03T17:29:31.223Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.385824902Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156982024-02-03T17:29:31.223Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.386711196Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
156992024-02-03T17:29:31.224Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.386860285Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157002024-02-03T17:29:31.224Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.386992423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157012024-02-03T17:29:31.224Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.387019997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157022024-02-03T17:29:31.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.38785901Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157032024-02-03T17:29:31.224Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.387932769Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157042024-02-03T17:29:31.224Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.387959119Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157052024-02-03T17:29:31.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.388007042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157062024-02-03T17:29:31.224Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.388031882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157072024-02-03T17:29:31.224Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.388053761Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157082024-02-03T17:29:31.224Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.388086752Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157092024-02-03T17:29:31.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.388891756Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157102024-02-03T17:29:31.224Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.388971989Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157112024-02-03T17:29:31.224Z{"msg":"[0]R flush_numbers[0..12]: [261, 262, 260, 267, 268, 268, 257, 267, 265, 252, 239, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389010568Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157122024-02-03T17:29:31.225Z{"msg":"[0]R generation[0..12]: [142, 142, 141, 144, 145, 145, 140, 144, 144, 137, 131, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389064064Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157132024-02-03T17:29:31.235Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389223373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157142024-02-03T17:29:31.235Z{"msg":"[1]R flush_numbers[0..12]: [261, 262, 260, 267, 268, 268, 257, 267, 265, 252, 239, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.38927629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157152024-02-03T17:29:31.236Z{"msg":"[1]R generation[0..12]: [142, 142, 141, 144, 145, 145, 140, 144, 144, 137, 131, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389302959Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157162024-02-03T17:29:31.236Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.38934607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157172024-02-03T17:29:31.236Z{"msg":"[2]R flush_numbers[0..12]: [261, 262, 260, 267, 268, 268, 257, 267, 265, 252, 239, 262]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389371841Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157182024-02-03T17:29:31.236Z{"msg":"[2]R generation[0..12]: [142, 142, 141, 144, 145, 145, 140, 144, 144, 137, 131, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389413907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157192024-02-03T17:29:31.236Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389439613Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157202024-02-03T17:29:31.236Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389461656Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157212024-02-03T17:29:31.236Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389484097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157222024-02-03T17:29:31.236ZWait for a query_work_queue command to finish before sending IO
157232024-02-03T17:29:31.236Z{"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389531701Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157242024-02-03T17:29:31.236Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389555715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157252024-02-03T17:29:31.236Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389577874Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157262024-02-03T17:29:31.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389619205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157272024-02-03T17:29:31.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389643028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157282024-02-03T17:29:31.237Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389683295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157292024-02-03T17:29:31.237Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389707148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157302024-02-03T17:29:31.237Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389727876Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157312024-02-03T17:29:31.237Z{"msg":"92c7a36b-3cf5-4a23-b4e2-2036e80e3348 is now active with session: c8131774-9b8d-4b96-b175-544c5d8a7ece","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389775264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157322024-02-03T17:29:31.237Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389796746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719,"session_id":"c8131774-9b8d-4b96-b175-544c5d8a7ece"}
157332024-02-03T17:29:31.237Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:30.389817784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719}
157342024-02-03T17:29:31.237ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157352024-02-03T17:29:31.237ZLoading write count information from file "/tmp/test_repair_verify.data"
157362024-02-03T17:29:31.237ZRun Repair workload
157372024-02-03T17:29:31.237Z01/30 Write block 253 len 36864 data: 10 12 12 11 9 9 8 10 11
157382024-02-03T17:29:31.237Z02/30 Read block 123 len 32768
157392024-02-03T17:29:31.238Z03/30 Write block 442 len 28672 data: 11 11 10 12 11 9 10
157402024-02-03T17:29:31.238Z04/30 Read block 358 len 8192
157412024-02-03T17:29:31.238Z05/30 Read block 586 len 28672
157422024-02-03T17:29:31.238Z06/30 Read block 423 len 12288
157432024-02-03T17:29:31.246Z07/30 Read block 520 len 12288
157442024-02-03T17:29:31.246Z08/30 Write block 34 len 28672 data: 6 5 6 7 8 9 9
157452024-02-03T17:29:31.246Z09/30 Read block 316 len 32768
157462024-02-03T17:29:31.246Z10/30 Read block 324 len 40960
157472024-02-03T17:29:31.246Z11/30 Read block 99 len 36864
157482024-02-03T17:29:31.246Z12/30 Flush
157492024-02-03T17:29:31.247Z13/30 Write block 585 len 4096 data: 13
157502024-02-03T17:29:31.247Z14/30 Flush
157512024-02-03T17:29:31.253Z15/30 Write block 371 len 28672 data: 9 8 6 7 8 11 12
157522024-02-03T17:29:31.253Z16/30 Read block 455 len 16384
157532024-02-03T17:29:31.253Z17/30 Write block 143 len 28672 data: 7 7 7 6 5 6 7
157542024-02-03T17:29:31.254Z18/30 Write block 278 len 40960 data: 18 14 14 16 17 15 14 11 10 11
157552024-02-03T17:29:31.254Z19/30 Write block 456 len 36864 data: 13 15 16 16 14 12 14 14 13
157562024-02-03T17:29:31.254Z20/30 Read block 509 len 24576
157572024-02-03T17:29:31.259Z21/30 Read block 176 len 20480
157582024-02-03T17:29:31.259Z22/30 Read block 588 len 32768
157592024-02-03T17:29:31.259Z23/30 Flush
157602024-02-03T17:29:31.262Z24/30 Write block 219 len 12288 data: 10 11 16
157612024-02-03T17:29:31.263Z25/30 Read block 17 len 12288
157622024-02-03T17:29:31.263Z26/30 Read block 372 len 20480
157632024-02-03T17:29:31.269Z27/30 Read block 559 len 36864
157642024-02-03T17:29:31.269Z28/30 Read block 154 len 36864
157652024-02-03T17:29:31.284Z29/30 Read block 505 len 12288
157662024-02-03T17:29:31.285Z30/30 Write block 208 len 40960 data: 12 10 11 9 8 8 9 10 8 9
157672024-02-03T17:29:31.285Z----------------------------------------------------------------
157682024-02-03T17:29:31.285Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
157692024-02-03T17:29:31.285ZStates: Active Active Active
157702024-02-03T17:29:31.285ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157712024-02-03T17:29:31.285Z 1 Acked 1000 Write 9 Done Done Sent false
157722024-02-03T17:29:31.285Z 2 Acked 1001 Read 8 Done Done Sent false
157732024-02-03T17:29:31.286Z 3 Acked 1002 Write 7 Done Done Sent false
157742024-02-03T17:29:31.286Z 4 Acked 1003 Read 2 Done Done Sent false
157752024-02-03T17:29:31.286Z 5 Acked 1004 Read 7 Done Done Sent false
157762024-02-03T17:29:31.286Z 6 Acked 1005 Read 3 Done Done Sent false
157772024-02-03T17:29:31.286Z 7 Acked 1006 Read 3 Done Done Sent false
157782024-02-03T17:29:31.286Z 8 Acked 1007 Write 7 Done Done Sent false
157792024-02-03T17:29:31.286Z 9 Acked 1008 Read 8 Done Done Sent false
157802024-02-03T17:29:31.286Z 10 Acked 1009 Read 10 Done Done Sent false
157812024-02-03T17:29:31.286Z 11 Acked 1010 Read 9 Done Done Sent false
157822024-02-03T17:29:31.286Z 12 Acked 1011 Flush 0 Done Done Sent false
157832024-02-03T17:29:31.287Z 13 Acked 1012 Write 1 Done Done Sent false
157842024-02-03T17:29:31.287Z 14 Acked 1013 Flush 0 Done Done Sent false
157852024-02-03T17:29:31.287Z 15 Acked 1014 Write 7 Done Done Sent false
157862024-02-03T17:29:31.287Z 16 Acked 1015 Read 4 Done Done Sent false
157872024-02-03T17:29:31.287Z 17 Acked 1016 Write 7 Done Done Sent false
157882024-02-03T17:29:31.287Z 18 Acked 1017 Write 10 Done Done Sent false
157892024-02-03T17:29:31.287Z 19 Acked 1018 Write 9 Done Done Sent false
157902024-02-03T17:29:31.288Z 20 Acked 1019 Read 6 Done Done Sent false
157912024-02-03T17:29:31.288Z 21 Acked 1020 Read 5 Done Done Sent false
157922024-02-03T17:29:31.288Z 22 Acked 1021 Read 8 Done Done Sent false
157932024-02-03T17:29:31.288Z 23 Acked 1022 Flush 0 Done Done Sent false
157942024-02-03T17:29:31.288Z 24 Acked 1023 Write 3 Done Done Sent false
157952024-02-03T17:29:31.288Z 25 Acked 1024 Read 3 Done Done Sent false
157962024-02-03T17:29:31.289Z 26 Acked 1025 Read 5 Done Done Sent false
157972024-02-03T17:29:31.289Z 27 Acked 1026 Read 9 Done Done Sent false
157982024-02-03T17:29:31.289Z 28 Acked 1027 Read 9 Done Done Sent false
157992024-02-03T17:29:31.289Z 29 Acked 1028 Read 3 Done Done Sent false
158002024-02-03T17:29:31.289Z 30 Acked 1029 Write 10 Sent Sent Sent false
158012024-02-03T17:29:31.289Z STATES DS:0 DS:1 DS:2 TOTAL
158022024-02-03T17:29:31.290Z New 0 0 0 0
158032024-02-03T17:29:31.290Z Sent 1 1 30 32
158042024-02-03T17:29:31.290Z Done 29 29 0 58
158052024-02-03T17:29:31.290Z Skipped 0 0 0 0
158062024-02-03T17:29:31.290Z Error 0 0 0 0
158072024-02-03T17:29:31.290ZLast Flush: 1022 1022 0
158082024-02-03T17:29:31.291ZDownstairs last five completed:
158092024-02-03T17:29:31.291ZUpstairs last five completed: 30 29 28 27 26
158102024-02-03T17:29:31.291Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:30.443472513Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1719}
158112024-02-03T17:29:31.291ZWrote out file "/tmp/test_repair_verify.data"
158122024-02-03T17:29:31.305Z
158132024-02-03T17:29:31.305ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158142024-02-03T17:29:31.315ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158152024-02-03T17:29:31.315Z 0 000-019 142 142 142 261 261 261 F F F
158162024-02-03T17:29:31.321Z 1 020-039 146 146 142 269 269 262 F F F <---
158172024-02-03T17:29:31.321Z 2 040-059 146 146 141 269 269 260 F F F <---
158182024-02-03T17:29:31.321Z 3 060-079 144 144 144 267 267 267 F F F
158192024-02-03T17:29:31.322Z 4 080-099 145 145 145 268 268 268 F F F
158202024-02-03T17:29:31.322Z 5 100-119 145 145 145 268 268 268 F F F
158212024-02-03T17:29:31.322Z 6 120-139 140 140 140 257 257 257 F F F
158222024-02-03T17:29:31.322Z 7 140-159 146 146 144 271 271 267 F F F <---
158232024-02-03T17:29:31.323Z 8 160-179 144 144 144 265 265 265 F F F
158242024-02-03T17:29:31.323Z 9 180-199 137 137 137 252 252 252 F F F
158252024-02-03T17:29:31.323Z 10 200-219 131 131 131 239 239 239 T T F <---
158262024-02-03T17:29:31.323Z 11 220-239 142 142 142 262 262 262 T T F <---
158272024-02-03T17:29:31.323Z 12 240-259 146 146 144 269 269 264 F F F <---
158282024-02-03T17:29:31.323Z 13 260-279 146 146 144 271 271 265 F F F <---
158292024-02-03T17:29:31.324Z 14 280-299 146 146 144 271 271 265 F F F <---
158302024-02-03T17:29:31.324Z 15 300-319 145 145 145 268 268 268 F F F
158312024-02-03T17:29:31.324Z 16 320-339 140 140 140 259 259 259 F F F
158322024-02-03T17:29:31.324Z 17 340-359 138 138 138 255 255 255 F F F
158332024-02-03T17:29:31.324Z 18 360-379 146 146 140 271 271 258 F F F <---
158342024-02-03T17:29:31.324Z 19 380-399 145 145 145 268 268 268 F F F
158352024-02-03T17:29:31.324Z 20 400-419 145 145 145 268 268 268 F F F
158362024-02-03T17:29:31.325Z 21 420-439 142 142 142 262 262 262 F F F
158372024-02-03T17:29:31.325Z 22 440-459 146 146 141 271 271 260 F F F <---
158382024-02-03T17:29:31.325Z 23 460-479 146 146 143 271 271 263 F F F <---
158392024-02-03T17:29:31.325Z 24 480-499 139 139 139 256 256 256 F F F
158402024-02-03T17:29:31.325Z 25 500-519 142 142 142 261 261 261 F F F
158412024-02-03T17:29:31.325Z 26 520-539 145 145 145 268 268 268 F F F
158422024-02-03T17:29:31.325Z 27 540-559 144 144 144 266 266 266 F F F
158432024-02-03T17:29:31.326Z 28 560-579 144 144 144 266 266 266 F F F
158442024-02-03T17:29:31.326Z 29 580-599 146 146 145 270 270 268 F F F <---
158452024-02-03T17:29:31.326ZMax gen: 146, Max flush: 271
158462024-02-03T17:29:31.326ZError: Difference in extent metadata found!
158472024-02-03T17:29:31.326ZA Difference in extent metadata is expected here
158482024-02-03T17:29:31.326ZOn loop 73
158492024-02-03T17:29:31.326Z
158502024-02-03T17:29:31.332ZVerifying data now
158512024-02-03T17:29:36.377ZNo change after verify
158522024-02-03T17:29:36.378ZLoop: 73 Downstairs dump after verify (and repair):
158532024-02-03T17:29:36.405ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158542024-02-03T17:29:36.405Z 0 000-019 142 142 142 261 261 261 F F F
158552024-02-03T17:29:36.406Z 1 020-039 146 146 146 269 269 269 F F F
158562024-02-03T17:29:36.406Z 2 040-059 146 146 146 269 269 269 F F F
158572024-02-03T17:29:36.406Z 3 060-079 144 144 144 267 267 267 F F F
158582024-02-03T17:29:36.406Z 4 080-099 145 145 145 268 268 268 F F F
158592024-02-03T17:29:36.406Z 5 100-119 145 145 145 268 268 268 F F F
158602024-02-03T17:29:36.406Z 6 120-139 140 140 140 257 257 257 F F F
158612024-02-03T17:29:36.418Z 7 140-159 146 146 146 271 271 271 F F F
158622024-02-03T17:29:36.419Z 8 160-179 144 144 144 265 265 265 F F F
158632024-02-03T17:29:36.419Z 9 180-199 137 137 137 252 252 252 F F F
158642024-02-03T17:29:36.419Z 10 200-219 147 147 147 272 272 272 F F F
158652024-02-03T17:29:36.419Z 11 220-239 147 147 147 272 272 272 F F F
158662024-02-03T17:29:36.419Z 12 240-259 146 146 146 269 269 269 F F F
158672024-02-03T17:29:36.419Z 13 260-279 146 146 146 271 271 271 F F F
158682024-02-03T17:29:36.419Z 14 280-299 146 146 146 271 271 271 F F F
158692024-02-03T17:29:36.419Z 15 300-319 145 145 145 268 268 268 F F F
158702024-02-03T17:29:36.419Z 16 320-339 140 140 140 259 259 259 F F F
158712024-02-03T17:29:36.420Z 17 340-359 138 138 138 255 255 255 F F F
158722024-02-03T17:29:36.420Z 18 360-379 146 146 146 271 271 271 F F F
158732024-02-03T17:29:36.420Z 19 380-399 145 145 145 268 268 268 F F F
158742024-02-03T17:29:36.420Z 20 400-419 145 145 145 268 268 268 F F F
158752024-02-03T17:29:36.420Z 21 420-439 142 142 142 262 262 262 F F F
158762024-02-03T17:29:36.420Z 22 440-459 146 146 146 271 271 271 F F F
158772024-02-03T17:29:36.420Z 23 460-479 146 146 146 271 271 271 F F F
158782024-02-03T17:29:36.420Z 24 480-499 139 139 139 256 256 256 F F F
158792024-02-03T17:29:36.420Z 25 500-519 142 142 142 261 261 261 F F F
158802024-02-03T17:29:36.420Z 26 520-539 145 145 145 268 268 268 F F F
158812024-02-03T17:29:36.421Z 27 540-559 144 144 144 266 266 266 F F F
158822024-02-03T17:29:36.421Z 28 560-579 144 144 144 266 266 266 F F F
158832024-02-03T17:29:36.421Z 29 580-599 146 146 146 270 270 270 F F F
158842024-02-03T17:29:36.421ZMax gen: 147, Max flush: 272
158852024-02-03T17:29:36.421Z
158862024-02-03T17:29:36.421ZBegin loop 74
158872024-02-03T17:29:36.421ZDownstairs to restart: 1
158882024-02-03T17:29:36.446Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.61536965Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727}
158892024-02-03T17:29:36.460Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.615997781Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727}
158902024-02-03T17:29:36.460Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.616114923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727}
158912024-02-03T17:29:36.461Z{"msg":"Crucible 5c0785e5-b769-428b-9093-be820d6a7109 has session id: 77547f9c-e994-4e7e-8a29-73ccb7f2377e","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
158922024-02-03T17:29:36.461Ztime":"2024-02-03T17:29:35.616193864Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
158932024-02-03T17:29:36.461Z{"msg":"Upstairs opts: Upstairs UUID: 5c0785e5-b769-428b-9093-be820d6a7109, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.616342265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
158942024-02-03T17:29:36.461Z{"msg":"Crucible stats registered with UUID: 5c0785e5-b769-428b-9093-be820d6a7109","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.616391413Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
158952024-02-03T17:29:36.462Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.616653422Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727}
158962024-02-03T17:29:36.462Z{"msg":"5c0785e5-b769-428b-9093-be820d6a7109 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.616785484Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
158972024-02-03T17:29:36.462Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.616862564Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
158982024-02-03T17:29:36.462Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.616911027Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
158992024-02-03T17:29:36.462Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.616954968Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159002024-02-03T17:29:36.462Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617029853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159012024-02-03T17:29:36.463Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617076073Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159022024-02-03T17:29:36.463Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617119388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159032024-02-03T17:29:36.463Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617159209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159042024-02-03T17:29:36.463Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617197985Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159052024-02-03T17:29:36.463Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617326768Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159062024-02-03T17:29:36.464Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617407702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159072024-02-03T17:29:36.464Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617482746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159082024-02-03T17:29:36.464Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.617656757Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159092024-02-03T17:29:36.464Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.618929017Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159102024-02-03T17:29:36.464Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.619041221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159112024-02-03T17:29:36.465Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.619137702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159122024-02-03T17:29:36.465Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.619186531Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159132024-02-03T17:29:36.465Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.620027506Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159142024-02-03T17:29:36.465Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.620175818Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159152024-02-03T17:29:36.465Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.620254947Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159162024-02-03T17:29:36.466Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.620329967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159172024-02-03T17:29:36.466Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.621094317Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159182024-02-03T17:29:36.466Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.62135793Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159192024-02-03T17:29:36.466Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.621430903Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159202024-02-03T17:29:36.467Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.621465685Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159212024-02-03T17:29:36.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.621489065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159222024-02-03T17:29:36.467Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.62151199Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159232024-02-03T17:29:36.467Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.621556201Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159242024-02-03T17:29:36.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.621929946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159252024-02-03T17:29:36.468Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.621983616Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159262024-02-03T17:29:36.468Z{"msg":"[0]R flush_numbers[0..12]: [261, 269, 269, 267, 268, 268, 257, 271, 265, 252, 272, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622010911Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159272024-02-03T17:29:36.468Z{"msg":"[0]R generation[0..12]: [142, 146, 146, 144, 145, 145, 140, 146, 144, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622053203Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159282024-02-03T17:29:36.468Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622079253Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159292024-02-03T17:29:36.468Z{"msg":"[1]R flush_numbers[0..12]: [261, 269, 269, 267, 268, 268, 257, 271, 265, 252, 272, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622121204Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159302024-02-03T17:29:36.469Z{"msg":"[1]R generation[0..12]: [142, 146, 146, 144, 145, 145, 140, 146, 144, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622147073Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159312024-02-03T17:29:36.469Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622189369Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159322024-02-03T17:29:36.469Z{"msg":"[2]R flush_numbers[0..12]: [261, 269, 269, 267, 268, 268, 257, 271, 265, 252, 272, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622215178Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159332024-02-03T17:29:36.469Z{"msg":"[2]R generation[0..12]: [142, 146, 146, 144, 145, 145, 140, 146, 144, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622256774Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159342024-02-03T17:29:36.470Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622282841Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159352024-02-03T17:29:36.470Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
159362024-02-03T17:29:36.470Z,"time":"2024-02-03T17:29:35.622323356Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159372024-02-03T17:29:36.470Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.62234788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159382024-02-03T17:29:36.470Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.62238788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159392024-02-03T17:29:36.471Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622412003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159402024-02-03T17:29:36.471Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622454044Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159412024-02-03T17:29:36.471Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.62247641Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159422024-02-03T17:29:36.471Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622522648Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159432024-02-03T17:29:36.471Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622561431Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159442024-02-03T17:29:36.472Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622604872Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159452024-02-03T17:29:36.472Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622664078Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159462024-02-03T17:29:36.472Z{"msg":"5c0785e5-b769-428b-9093-be820d6a7109 is now active with session: 77547f9c-e994-4e7e-8a29-73ccb7f2377e","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622689624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159472024-02-03T17:29:36.472Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622720018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727,"session_id":"77547f9c-e994-4e7e-8a29-73ccb7f2377e"}
159482024-02-03T17:29:36.472Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:35.622757972Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727}
159492024-02-03T17:29:36.473ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159502024-02-03T17:29:36.473ZLoading write count information from file "/tmp/test_repair_verify.data"
159512024-02-03T17:29:36.473ZRun Repair workload
159522024-02-03T17:29:36.484Z01/30 Write block 47 len 16384 data: 5 9 9 12
159532024-02-03T17:29:36.485Z02/30 Write block 382 len 40960 data: 9 9 9 9 7 8 9 10 11 11
159542024-02-03T17:29:36.485Z03/30 Write block 260 len 8192 data: 11 12
159552024-02-03T17:29:36.485Z04/30 Write block 489 len 12288 data: 10 9 10
159562024-02-03T17:29:36.485Z05/30 Write block 527 len 32768 data: 13 13 14 12 11 11 8 7
159572024-02-03T17:29:36.485Z06/30 Read block 363 len 12288
159582024-02-03T17:29:36.486Z07/30 Flush
159592024-02-03T17:29:36.486Z08/30 Write block 373 len 8192 data: 7 8
159602024-02-03T17:29:36.486Z09/30 Read block 136 len 4096
159612024-02-03T17:29:36.486Z10/30 Read block 10 len 36864
159622024-02-03T17:29:36.486Z11/30 Write block 123 len 32768 data: 10 7 8 11 11 15 17 18
159632024-02-03T17:29:36.487Z12/30 Write block 120 len 28672 data: 14 15 11 11 8 9 12
159642024-02-03T17:29:36.487Z13/30 Read block 41 len 40960
159652024-02-03T17:29:36.487Z14/30 Write block 100 len 40960 data: 12 12 11 13 11 14 11 12 13 12
159662024-02-03T17:29:36.487Z15/30 Read block 184 len 32768
159672024-02-03T17:29:36.487Z16/30 Flush
159682024-02-03T17:29:36.487Z17/30 Flush
159692024-02-03T17:29:36.488Z18/30 Write block 587 len 20480 data: 13 13 15 14 14
159702024-02-03T17:29:36.505Z19/30 Write block 176 len 4096 data: 11
159712024-02-03T17:29:36.505Z20/30 Read block 500 len 24576
159722024-02-03T17:29:36.505Z21/30 Read block 352 len 40960
159732024-02-03T17:29:36.505Z22/30 Write block 471 len 16384 data: 7 6 5 6
159742024-02-03T17:29:36.505Z23/30 Read block 567 len 8192
159752024-02-03T17:29:36.505Z24/30 Write block 343 len 12288 data: 9 10 10
159762024-02-03T17:29:36.505Z25/30 Read block 271 len 36864
159772024-02-03T17:29:36.505Z26/30 Read block 273 len 16384
159782024-02-03T17:29:36.505Z27/30 Write block 2 len 36864 data: 7 8 10 12 12 14 14 13 17
159792024-02-03T17:29:36.528Z28/30 Write block 512 len 36864 data: 12 10 7 9 7 10 9 6 6
159802024-02-03T17:29:36.528Z29/30 Read block 518 len 16384
159812024-02-03T17:29:36.528Z30/30 Read block 408 len 32768
159822024-02-03T17:29:36.528Z----------------------------------------------------------------
159832024-02-03T17:29:36.528Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
159842024-02-03T17:29:36.528ZStates: Active Active Active
159852024-02-03T17:29:36.528ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159862024-02-03T17:29:36.528Z 1 Acked 1000 Write 4 Done Sent Done false
159872024-02-03T17:29:36.529Z 2 Acked 1001 Write 10 Done Sent Done false
159882024-02-03T17:29:36.529Z 3 Acked 1002 Write 2 Done Sent Done false
159892024-02-03T17:29:36.529Z 4 Acked 1003 Write 3 Done Sent Done false
159902024-02-03T17:29:36.529Z 5 Acked 1004 Write 8 Done Sent Done false
159912024-02-03T17:29:36.529Z 6 Acked 1005 Read 3 Done Sent Done false
159922024-02-03T17:29:36.529Z 7 Acked 1006 Flush 0 Done Sent Done false
159932024-02-03T17:29:36.530Z 8 Acked 1007 Write 2 Done Sent Done false
159942024-02-03T17:29:36.530Z 9 Acked 1008 Read 1 Done Sent Done false
159952024-02-03T17:29:36.530Z 10 Acked 1009 Read 9 Done Sent Done false
159962024-02-03T17:29:36.530Z 11 Acked 1010 Write 8 Done Sent Done false
159972024-02-03T17:29:36.530Z 12 Acked 1011 Write 7 Done Sent Done false
159982024-02-03T17:29:36.530Z 13 Acked 1012 Read 10 Done Sent Done false
159992024-02-03T17:29:36.530Z 14 Acked 1013 Write 10 Done Sent Done false
160002024-02-03T17:29:36.530Z 15 Acked 1014 Read 8 Done Sent Done false
160012024-02-03T17:29:36.530Z 16 Acked 1015 Flush 0 Done Sent Done false
160022024-02-03T17:29:36.531Z 17 Acked 1016 Flush 0 Done Sent Done false
160032024-02-03T17:29:36.531Z 18 Acked 1017 Write 5 Done Sent Done false
160042024-02-03T17:29:36.531Z 19 Acked 1018 Write 1 Done Sent Done false
160052024-02-03T17:29:36.531Z 20 Acked 1019 Read 6 Done Sent Done false
160062024-02-03T17:29:36.531Z 21 Acked 1020 Read 10 Done Sent Done false
160072024-02-03T17:29:36.531Z 22 Acked 1021 Write 4 Done Sent Done false
160082024-02-03T17:29:36.531Z 23 Acked 1022 Read 2 Done Sent Done false
160092024-02-03T17:29:36.531Z 24 Acked 1023 Write 3 Done Sent Done false
160102024-02-03T17:29:36.532Z 25 Acked 1024 Read 9 Done Sent Done false
160112024-02-03T17:29:36.532Z 26 Acked 1025 Read 4 Done Sent Done false
160122024-02-03T17:29:36.532Z 27 Acked 1026 Write 9 Done Sent Done false
160132024-02-03T17:29:36.532Z 28 Acked 1027 Write 9 Done Sent Done false
160142024-02-03T17:29:36.532Z 29 Acked 1028 Read 4 Done Sent Done false
160152024-02-03T17:29:36.532Z 30 Acked 1029 Read 8 Done Sent Sent false
160162024-02-03T17:29:36.532Z STATES DS:0 DS:1 DS:2 TOTAL
160172024-02-03T17:29:36.532Z New 0 0 0 0
160182024-02-03T17:29:36.532Z Sent 0 30 1 31
160192024-02-03T17:29:36.532Z Done 30 0 29 59
160202024-02-03T17:29:36.533Z Skipped 0 0 0 0
160212024-02-03T17:29:36.533Z Error 0 0 0 0
160222024-02-03T17:29:36.533ZLast Flush: 1016 0 1016
160232024-02-03T17:29:36.533ZDownstairs last five completed:
160242024-02-03T17:29:36.533ZUpstairs last five completed: 30 29 28 27 26
160252024-02-03T17:29:36.533Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:35.670834267Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1727}
160262024-02-03T17:29:36.533ZWrote out file "/tmp/test_repair_verify.data"
160272024-02-03T17:29:36.533Z
160282024-02-03T17:29:36.538ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160292024-02-03T17:29:36.554ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160302024-02-03T17:29:36.554Z 0 000-019 142 142 142 261 261 261 T F T <---
160312024-02-03T17:29:36.555Z 1 020-039 146 146 146 269 269 269 F F F
160322024-02-03T17:29:36.555Z 2 040-059 148 146 148 273 269 273 F F F <---
160332024-02-03T17:29:36.555Z 3 060-079 144 144 144 267 267 267 F F F
160342024-02-03T17:29:36.555Z 4 080-099 145 145 145 268 268 268 F F F
160352024-02-03T17:29:36.555Z 5 100-119 148 145 148 274 268 274 F F F <---
160362024-02-03T17:29:36.555Z 6 120-139 148 140 148 274 257 274 F F F <---
160372024-02-03T17:29:36.555Z 7 140-159 146 146 146 271 271 271 F F F
160382024-02-03T17:29:36.555Z 8 160-179 144 144 144 265 265 265 T F T <---
160392024-02-03T17:29:36.555Z 9 180-199 137 137 137 252 252 252 F F F
160402024-02-03T17:29:36.556Z 10 200-219 147 147 147 272 272 272 F F F
160412024-02-03T17:29:36.556Z 11 220-239 147 147 147 272 272 272 F F F
160422024-02-03T17:29:36.556Z 12 240-259 146 146 146 269 269 269 F F F
160432024-02-03T17:29:36.556Z 13 260-279 148 146 148 273 271 273 F F F <---
160442024-02-03T17:29:36.556Z 14 280-299 146 146 146 271 271 271 F F F
160452024-02-03T17:29:36.556Z 15 300-319 145 145 145 268 268 268 F F F
160462024-02-03T17:29:36.556Z 16 320-339 140 140 140 259 259 259 F F F
160472024-02-03T17:29:36.556Z 17 340-359 138 138 138 255 255 255 T F T <---
160482024-02-03T17:29:36.556Z 18 360-379 148 146 148 274 271 274 F F F <---
160492024-02-03T17:29:36.556Z 19 380-399 148 145 148 273 268 273 F F F <---
160502024-02-03T17:29:36.557Z 20 400-419 145 145 145 268 268 268 F F F
160512024-02-03T17:29:36.557Z 21 420-439 142 142 142 262 262 262 F F F
160522024-02-03T17:29:36.557Z 22 440-459 146 146 146 271 271 271 F F F
160532024-02-03T17:29:36.557Z 23 460-479 146 146 146 271 271 271 T F T <---
160542024-02-03T17:29:36.557Z 24 480-499 148 139 148 273 256 273 F F F <---
160552024-02-03T17:29:36.557Z 25 500-519 142 142 142 261 261 261 T F T <---
160562024-02-03T17:29:36.557Z 26 520-539 148 145 148 273 268 273 T F T <---
160572024-02-03T17:29:36.557Z 27 540-559 144 144 144 266 266 266 F F F
160582024-02-03T17:29:36.557Z 28 560-579 144 144 144 266 266 266 F F F
160592024-02-03T17:29:36.557Z 29 580-599 146 146 146 270 270 270 T F T <---
160602024-02-03T17:29:36.557ZMax gen: 148, Max flush: 274
160612024-02-03T17:29:36.558ZError: Difference in extent metadata found!
160622024-02-03T17:29:36.562ZA Difference in extent metadata is expected here
160632024-02-03T17:29:36.562ZOn loop 74
160642024-02-03T17:29:36.562Z
160652024-02-03T17:29:36.562ZVerifying data now
160662024-02-03T17:29:41.779ZNo change after verify
160672024-02-03T17:29:41.779ZLoop: 74 Downstairs dump after verify (and repair):
160682024-02-03T17:29:41.801ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160692024-02-03T17:29:41.801Z 0 000-019 149 149 149 275 275 275 F F F
160702024-02-03T17:29:41.805Z 1 020-039 146 146 146 269 269 269 F F F
160712024-02-03T17:29:41.805Z 2 040-059 148 148 148 273 273 273 F F F
160722024-02-03T17:29:41.805Z 3 060-079 144 144 144 267 267 267 F F F
160732024-02-03T17:29:41.805Z 4 080-099 145 145 145 268 268 268 F F F
160742024-02-03T17:29:41.805Z 5 100-119 148 148 148 274 274 274 F F F
160752024-02-03T17:29:41.805Z 6 120-139 148 148 148 274 274 274 F F F
160762024-02-03T17:29:41.805Z 7 140-159 146 146 146 271 271 271 F F F
160772024-02-03T17:29:41.805Z 8 160-179 149 149 149 275 275 275 F F F
160782024-02-03T17:29:41.805Z 9 180-199 137 137 137 252 252 252 F F F
160792024-02-03T17:29:41.805Z 10 200-219 147 147 147 272 272 272 F F F
160802024-02-03T17:29:41.805Z 11 220-239 147 147 147 272 272 272 F F F
160812024-02-03T17:29:41.805Z 12 240-259 146 146 146 269 269 269 F F F
160822024-02-03T17:29:41.806Z 13 260-279 148 148 148 273 273 273 F F F
160832024-02-03T17:29:41.806Z 14 280-299 146 146 146 271 271 271 F F F
160842024-02-03T17:29:41.806Z 15 300-319 145 145 145 268 268 268 F F F
160852024-02-03T17:29:41.806Z 16 320-339 140 140 140 259 259 259 F F F
160862024-02-03T17:29:41.806Z 17 340-359 149 149 149 275 275 275 F F F
160872024-02-03T17:29:41.806Z 18 360-379 148 148 148 274 274 274 F F F
160882024-02-03T17:29:41.806Z 19 380-399 148 148 148 273 273 273 F F F
160892024-02-03T17:29:41.806Z 20 400-419 145 145 145 268 268 268 F F F
160902024-02-03T17:29:41.806Z 21 420-439 142 142 142 262 262 262 F F F
160912024-02-03T17:29:41.806Z 22 440-459 146 146 146 271 271 271 F F F
160922024-02-03T17:29:41.806Z 23 460-479 149 149 149 275 275 275 F F F
160932024-02-03T17:29:41.806Z 24 480-499 148 148 148 273 273 273 F F F
160942024-02-03T17:29:41.807Z 25 500-519 149 149 149 275 275 275 F F F
160952024-02-03T17:29:41.807Z 26 520-539 149 149 149 275 275 275 F F F
160962024-02-03T17:29:41.807Z 27 540-559 144 144 144 266 266 266 F F F
160972024-02-03T17:29:41.807Z 28 560-579 144 144 144 266 266 266 F F F
160982024-02-03T17:29:41.807Z 29 580-599 149 149 149 275 275 275 F F F
160992024-02-03T17:29:41.807ZMax gen: 149, Max flush: 275
161002024-02-03T17:29:41.811Z
161012024-02-03T17:29:41.811ZBegin loop 75
161022024-02-03T17:29:41.811ZDownstairs to restart: 1
161032024-02-03T17:29:41.847Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.012053723Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735}
161042024-02-03T17:29:41.847Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.012553541Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735}
161052024-02-03T17:29:41.861Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.012613747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735}
161062024-02-03T17:29:41.861Z{"msg":"Crucible d7d2680e-43d2-4de2-8d74-3a308712b8c6 has session id: 02eb0ce6-49ca-4777-967f-5b813ac4b1ec","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.012684287Z"Crucible runtime is spawned
161072024-02-03T17:29:41.861Z,"hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161082024-02-03T17:29:41.861Z{"msg":"Upstairs opts: Upstairs UUID: d7d2680e-43d2-4de2-8d74-3a308712b8c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.012834961Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161092024-02-03T17:29:41.861Z{"msg":"Crucible stats registered with UUID: d7d2680e-43d2-4de2-8d74-3a308712b8c6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.012884238Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161102024-02-03T17:29:41.861Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.012943352Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735}
161112024-02-03T17:29:41.861Z{"msg":"d7d2680e-43d2-4de2-8d74-3a308712b8c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013038953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161122024-02-03T17:29:41.861Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013117896Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161132024-02-03T17:29:41.861Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013191092Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161142024-02-03T17:29:41.861Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013301232Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161152024-02-03T17:29:41.862Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013347713Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161162024-02-03T17:29:41.862Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013475556Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161172024-02-03T17:29:41.862Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013583601Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161182024-02-03T17:29:41.862Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013630716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161192024-02-03T17:29:41.862Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.0137468Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161202024-02-03T17:29:41.862Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013831052Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161212024-02-03T17:29:41.862Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.01387974Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161222024-02-03T17:29:41.862Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013925007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161232024-02-03T17:29:41.862Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.013997566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161242024-02-03T17:29:41.862Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.015059756Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161252024-02-03T17:29:41.862Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.015150952Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161262024-02-03T17:29:41.862Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.015199884Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161272024-02-03T17:29:41.863Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.015245616Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161282024-02-03T17:29:41.863Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.016046803Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161292024-02-03T17:29:41.863Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.016357288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161302024-02-03T17:29:41.863Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.016483739Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161312024-02-03T17:29:41.863Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.016623209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161322024-02-03T17:29:41.863Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.016825993Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161332024-02-03T17:29:41.863Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.016916347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161342024-02-03T17:29:41.863Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.01696146Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161352024-02-03T17:29:41.863Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.017313388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161362024-02-03T17:29:41.863Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.017375789Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161372024-02-03T17:29:41.863Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.017402388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161382024-02-03T17:29:41.863Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.017444003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161392024-02-03T17:29:41.864Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018060425Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161402024-02-03T17:29:41.864Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018125858Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161412024-02-03T17:29:41.864Z{"msg":"[0]R flush_numbers[0..12]: [275, 269, 273, 267, 268, 274, 274, 271, 275, 252, 272, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018153786Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161422024-02-03T17:29:41.864Z{"msg":"[0]R generation[0..12]: [149, 146, 148, 144, 145, 148, 148, 146, 149, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018215861Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161432024-02-03T17:29:41.864Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018250875Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161442024-02-03T17:29:41.864Z{"msg":"[1]R flush_numbers[0..12]: [275, 269, 273, 267, 268, 274, 274, 271, 275, 252, 272, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018308039Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161452024-02-03T17:29:41.864Z{"msg":"[1]R generation[0..12]: [149, 146, 148, 144, 145, 148, 148, 146, 149, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018341149Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161462024-02-03T17:29:41.864Z{"msg":"Wait for a query_work_queue command to finish before sending IO
161472024-02-03T17:29:41.864Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018400121Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161482024-02-03T17:29:41.864Z{"msg":"[2]R flush_numbers[0..12]: [275, 269, 273, 267, 268, 274, 274, 271, 275, 252, 272, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018437875Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161492024-02-03T17:29:41.864Z{"msg":"[2]R generation[0..12]: [149, 146, 148, 144, 145, 148, 148, 146, 149, 137, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018498624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161502024-02-03T17:29:41.864Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018525687Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161512024-02-03T17:29:41.864Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018588577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161522024-02-03T17:29:41.865Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018617008Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161532024-02-03T17:29:41.865Z{"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.0186735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161542024-02-03T17:29:41.865Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018700864Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161552024-02-03T17:29:41.865Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018759995Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161562024-02-03T17:29:41.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018784362Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161572024-02-03T17:29:41.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018845839Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161582024-02-03T17:29:41.865Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018872271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161592024-02-03T17:29:41.865Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018906401Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161602024-02-03T17:29:41.865Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.018961899Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161612024-02-03T17:29:41.865Z{"msg":"d7d2680e-43d2-4de2-8d74-3a308712b8c6 is now active with session: 02eb0ce6-49ca-4777-967f-5b813ac4b1ec","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.01898743Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161622024-02-03T17:29:41.865Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.019013744Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161632024-02-03T17:29:41.865Z02eb0ce6-49ca-4777-967f-5b813ac4b1ec"}
161642024-02-03T17:29:41.865Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:41.019110145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735}
161652024-02-03T17:29:41.866ZLoading write count information from file "/tmp/test_repair_verify.data"
161662024-02-03T17:29:41.866ZRun Repair workload
161672024-02-03T17:29:41.866Z01/30 Write block 158 len 8192 data: 13 13
161682024-02-03T17:29:41.866Z02/30 Write block 489 len 20480 data: 11 10 11 10 13
161692024-02-03T17:29:41.866Z03/30 Flush
161702024-02-03T17:29:41.875Z04/30 Write block 256 len 8192 data: 12 10
161712024-02-03T17:29:41.875Z05/30 Read block 526 len 20480
161722024-02-03T17:29:41.876Z06/30 Read block 549 len 20480
161732024-02-03T17:29:41.876Z07/30 Write block 215 len 8192 data: 11 9
161742024-02-03T17:29:41.876Z08/30 Read block 206 len 32768
161752024-02-03T17:29:41.876Z09/30 Flush
161762024-02-03T17:29:41.876Z10/30 Write block 577 len 40960 data: 13 16 16 17 17 17 15 16 14 17
161772024-02-03T17:29:41.876Z11/30 Write block 16 len 4096 data: 11
161782024-02-03T17:29:41.876Z12/30 Write block 478 len 36864 data: 11 10 9 11 10 9 7 7 9
161792024-02-03T17:29:41.876Z13/30 Write block 450 len 12288 data: 8 7 8
161802024-02-03T17:29:41.876Z14/30 Write block 144 len 8192 data: 8 8
161812024-02-03T17:29:41.876Z15/30 Read block 288 len 40960
161822024-02-03T17:29:41.876Z16/30 Write block 461 len 28672 data: 13 15 15 14 14 11 12
161832024-02-03T17:29:41.876Z17/30 Read block 203 len 40960
161842024-02-03T17:29:41.884Z18/30 Flush
161852024-02-03T17:29:41.884Z19/30 Write block 38 len 8192 data: 9 10
161862024-02-03T17:29:41.884Z20/30 Write block 304 len 40960 data: 10 13 14 11 13 13 11 10 7 9
161872024-02-03T17:29:41.885Z21/30 Write block 432 len 28672 data: 12 10 12 12 12 13 14
161882024-02-03T17:29:41.885Z22/30 Read block 383 len 36864
161892024-02-03T17:29:41.897Z23/30 Write block 2 len 20480 data: 8 9 11 13 13
161902024-02-03T17:29:41.897Z24/30 Read block 190 len 24576
161912024-02-03T17:29:41.897Z25/30 Read block 216 len 16384
161922024-02-03T17:29:41.897Z26/30 Read block 214 len 28672
161932024-02-03T17:29:41.897Z27/30 Write block 155 len 8192 data: 13 13
161942024-02-03T17:29:41.897Z28/30 Read block 136 len 24576
161952024-02-03T17:29:41.903Z29/30 Read block 513 len 32768
161962024-02-03T17:29:41.903Z30/30 Read block 504 len 16384
161972024-02-03T17:29:41.903Z----------------------------------------------------------------
161982024-02-03T17:29:41.903Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
161992024-02-03T17:29:41.903ZStates: Active Active Active
162002024-02-03T17:29:41.904ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162012024-02-03T17:29:41.904Z 1 Acked 1000 Write 2 Done Done Done false
162022024-02-03T17:29:41.904Z 2 Acked 1001 Write 5 Done Sent Done false
162032024-02-03T17:29:41.904Z 3 Acked 1002 Flush 0 Done Sent Done false
162042024-02-03T17:29:41.904Z 4 Acked 1003 Write 2 Done Sent Done false
162052024-02-03T17:29:41.904Z 5 Acked 1004 Read 5 Done Sent Done false
162062024-02-03T17:29:41.904Z 6 Acked 1005 Read 5 Done Sent Done false
162072024-02-03T17:29:41.904Z 7 Acked 1006 Write 2 Done Sent Done false
162082024-02-03T17:29:41.904Z 8 Acked 1007 Read 8 Done Sent Done false
162092024-02-03T17:29:41.904Z 9 Acked 1008 Flush 0 Done Sent Done false
162102024-02-03T17:29:41.904Z 10 Acked 1009 Write 10 Done Sent Done false
162112024-02-03T17:29:41.904Z 11 Acked 1010 Write 1 Done Sent Done false
162122024-02-03T17:29:41.904Z 12 Acked 1011 Write 9 Done Sent Done false
162132024-02-03T17:29:41.904Z 13 Acked 1012 Write 3 Done Sent Done false
162142024-02-03T17:29:41.904Z 14 Acked 1013 Write 2 Done Sent Done false
162152024-02-03T17:29:41.904Z 15 Acked 1014 Read 10 Done Sent Done false
162162024-02-03T17:29:41.904Z 16 Acked 1015 Write 7 Done Sent Done false
162172024-02-03T17:29:41.904Z 17 Acked 1016 Read 10 Done Sent Done false
162182024-02-03T17:29:41.904Z 18 Acked 1017 Flush 0 Done Sent Done false
162192024-02-03T17:29:41.904Z 19 Acked 1018 Write 2 Done Sent Done false
162202024-02-03T17:29:41.904Z 20 Acked 1019 Write 10 Done Sent Done false
162212024-02-03T17:29:41.904Z 21 Acked 1020 Write 7 Done Sent Done false
162222024-02-03T17:29:41.904Z 22 Acked 1021 Read 9 Done Sent Done false
162232024-02-03T17:29:41.904Z 23 Acked 1022 Write 5 Done Sent Done false
162242024-02-03T17:29:41.905Z 24 Acked 1023 Read 6 Done Sent Done false
162252024-02-03T17:29:41.905Z 25 Acked 1024 Read 4 Done Sent Done false
162262024-02-03T17:29:41.905Z 26 Acked 1025 Read 7 Done Sent Done false
162272024-02-03T17:29:41.905Z 27 Acked 1026 Write 2 Done Sent Done false
162282024-02-03T17:29:41.905Z 28 Acked 1027 Read 6 Done Sent Done false
162292024-02-03T17:29:41.905Z 29 Acked 1028 Read 8 Done Sent Done false
162302024-02-03T17:29:41.905Z 30 Acked 1029 Read 4 Sent Sent Done false
162312024-02-03T17:29:41.905Z STATES DS:0 DS:1 DS:2 TOTAL
162322024-02-03T17:29:41.905Z New 0 0 0 0
162332024-02-03T17:29:41.905Z Sent 1 29 0 30
162342024-02-03T17:29:41.905Z Done 29 1 30 60
162352024-02-03T17:29:41.905Z Skipped 0 0 0 0
162362024-02-03T17:29:41.905Z Error 0 0 0 0
162372024-02-03T17:29:41.905ZLast Flush: 1017 0 1017
162382024-02-03T17:29:41.905ZDownstairs last five completed:
162392024-02-03T17:29:41.905ZUpstairs last five completed: 30 29 28 27 26
162402024-02-03T17:29:41.905Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:41.06745558Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1735}
162412024-02-03T17:29:41.905ZWrote out file "/tmp/test_repair_verify.data"
162422024-02-03T17:29:41.905Z
162432024-02-03T17:29:41.913ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162442024-02-03T17:29:41.943ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162452024-02-03T17:29:41.943Z 0 000-019 150 149 150 278 275 278 T F T <---
162462024-02-03T17:29:41.943Z 1 020-039 146 146 146 269 269 269 T F T <---
162472024-02-03T17:29:41.943Z 2 040-059 148 148 148 273 273 273 F F F
162482024-02-03T17:29:41.943Z 3 060-079 144 144 144 267 267 267 F F F
162492024-02-03T17:29:41.943Z 4 080-099 145 145 145 268 268 268 F F F
162502024-02-03T17:29:41.950Z 5 100-119 148 148 148 274 274 274 F F F
162512024-02-03T17:29:41.950Z 6 120-139 148 148 148 274 274 274 F F F
162522024-02-03T17:29:41.950Z 7 140-159 150 146 150 278 271 278 T T T <---
162532024-02-03T17:29:41.950Z 8 160-179 149 149 149 275 275 275 F F F
162542024-02-03T17:29:41.950Z 9 180-199 137 137 137 252 252 252 F F F
162552024-02-03T17:29:41.951Z 10 200-219 150 147 150 277 272 277 F F F <---
162562024-02-03T17:29:41.951Z 11 220-239 147 147 147 272 272 272 F F F
162572024-02-03T17:29:41.951Z 12 240-259 150 146 150 277 269 277 F F F <---
162582024-02-03T17:29:41.951Z 13 260-279 148 148 148 273 273 273 F F F
162592024-02-03T17:29:41.951Z 14 280-299 146 146 146 271 271 271 F F F
162602024-02-03T17:29:41.951Z 15 300-319 145 145 145 268 268 268 T F T <---
162612024-02-03T17:29:41.951Z 16 320-339 140 140 140 259 259 259 F F F
162622024-02-03T17:29:41.951Z 17 340-359 149 149 149 275 275 275 F F F
162632024-02-03T17:29:41.951Z 18 360-379 148 148 148 274 274 274 F F F
162642024-02-03T17:29:41.951Z 19 380-399 148 148 148 273 273 273 F F F
162652024-02-03T17:29:41.952Z 20 400-419 145 145 145 268 268 268 F F F
162662024-02-03T17:29:41.952Z 21 420-439 142 142 142 262 262 262 T F T <---
162672024-02-03T17:29:41.952Z 22 440-459 150 146 150 278 271 278 F F F <---
162682024-02-03T17:29:41.952Z 23 460-479 150 149 150 278 275 278 F F F <---
162692024-02-03T17:29:41.952Z 24 480-499 150 148 150 278 273 278 F F F <---
162702024-02-03T17:29:41.952Z 25 500-519 149 149 149 275 275 275 F F F
162712024-02-03T17:29:41.952Z 26 520-539 149 149 149 275 275 275 F F F
162722024-02-03T17:29:41.952Z 27 540-559 144 144 144 266 266 266 F F F
162732024-02-03T17:29:41.952Z 28 560-579 150 144 150 278 266 278 F F F <---
162742024-02-03T17:29:41.952Z 29 580-599 150 149 150 278 275 278 F F F <---
162752024-02-03T17:29:41.952ZMax gen: 150, Max flush: 278
162762024-02-03T17:29:41.952ZError: Difference in extent metadata found!
162772024-02-03T17:29:41.952ZA Difference in extent metadata is expected here
162782024-02-03T17:29:41.952ZOn loop 75
162792024-02-03T17:29:41.953Z
162802024-02-03T17:29:41.953ZVerifying data now
162812024-02-03T17:29:47.007ZNo change after verify
162822024-02-03T17:29:47.008ZLoop: 75 Downstairs dump after verify (and repair):
162832024-02-03T17:29:47.032ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162842024-02-03T17:29:47.032Z 0 000-019 151 151 151 279 279 279 F F F
162852024-02-03T17:29:47.032Z 1 020-039 151 151 151 279 279 279 F F F
162862024-02-03T17:29:47.032Z 2 040-059 148 148 148 273 273 273 F F F
162872024-02-03T17:29:47.032Z 3 060-079 144 144 144 267 267 267 F F F
162882024-02-03T17:29:47.037Z 4 080-099 145 145 145 268 268 268 F F F
162892024-02-03T17:29:47.037Z 5 100-119 148 148 148 274 274 274 F F F
162902024-02-03T17:29:47.037Z 6 120-139 148 148 148 274 274 274 F F F
162912024-02-03T17:29:47.037Z 7 140-159 151 151 151 279 279 279 F F F
162922024-02-03T17:29:47.037Z 8 160-179 149 149 149 275 275 275 F F F
162932024-02-03T17:29:47.037Z 9 180-199 137 137 137 252 252 252 F F F
162942024-02-03T17:29:47.037Z 10 200-219 150 150 150 277 277 277 F F F
162952024-02-03T17:29:47.038Z 11 220-239 147 147 147 272 272 272 F F F
162962024-02-03T17:29:47.038Z 12 240-259 150 150 150 277 277 277 F F F
162972024-02-03T17:29:47.038Z 13 260-279 148 148 148 273 273 273 F F F
162982024-02-03T17:29:47.038Z 14 280-299 146 146 146 271 271 271 F F F
162992024-02-03T17:29:47.038Z 15 300-319 151 151 151 279 279 279 F F F
163002024-02-03T17:29:47.038Z 16 320-339 140 140 140 259 259 259 F F F
163012024-02-03T17:29:47.038Z 17 340-359 149 149 149 275 275 275 F F F
163022024-02-03T17:29:47.038Z 18 360-379 148 148 148 274 274 274 F F F
163032024-02-03T17:29:47.038Z 19 380-399 148 148 148 273 273 273 F F F
163042024-02-03T17:29:47.038Z 20 400-419 145 145 145 268 268 268 F F F
163052024-02-03T17:29:47.038Z 21 420-439 151 151 151 279 279 279 F F F
163062024-02-03T17:29:47.038Z 22 440-459 150 150 150 278 278 278 F F F
163072024-02-03T17:29:47.038Z 23 460-479 150 150 150 278 278 278 F F F
163082024-02-03T17:29:47.038Z 24 480-499 150 150 150 278 278 278 F F F
163092024-02-03T17:29:47.038Z 25 500-519 149 149 149 275 275 275 F F F
163102024-02-03T17:29:47.038Z 26 520-539 149 149 149 275 275 275 F F F
163112024-02-03T17:29:47.038Z 27 540-559 144 144 144 266 266 266 F F F
163122024-02-03T17:29:47.038Z 28 560-579 150 150 150 278 278 278 F F F
163132024-02-03T17:29:47.038Z 29 580-599 150 150 150 278 278 278 F F F
163142024-02-03T17:29:47.038ZMax gen: 151, Max flush: 279
163152024-02-03T17:29:47.042Z
163162024-02-03T17:29:47.042ZBegin loop 76
163172024-02-03T17:29:47.042ZDownstairs to restart: 2
163182024-02-03T17:29:47.071Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.24500833Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743}
163192024-02-03T17:29:47.071Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.245487946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743}
163202024-02-03T17:29:47.084Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.245555482Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743}
163212024-02-03T17:29:47.084Z{"msg":"Crucible 97a2cfa5-5c49-4d2a-a50b-84f4a5ebbae0 has session id: 29ee76a9-e12f-4b07-aa5f-3989890508de","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.245595992Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163222024-02-03T17:29:47.084Z{"msg":"Crucible runtime is spawned
163232024-02-03T17:29:47.084ZUpstairs opts: Upstairs UUID: 97a2cfa5-5c49-4d2a-a50b-84f4a5ebbae0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.245658309Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163242024-02-03T17:29:47.084Z{"msg":"Crucible stats registered with UUID: 97a2cfa5-5c49-4d2a-a50b-84f4a5ebbae0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.245702615Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163252024-02-03T17:29:47.084Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.245756688Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743}
163262024-02-03T17:29:47.084Z{"msg":"97a2cfa5-5c49-4d2a-a50b-84f4a5ebbae0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246057409Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163272024-02-03T17:29:47.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246164053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163282024-02-03T17:29:47.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246207973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163292024-02-03T17:29:47.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246246872Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163302024-02-03T17:29:47.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246316217Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163312024-02-03T17:29:47.084Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.24650039Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163322024-02-03T17:29:47.084Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246581922Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163332024-02-03T17:29:47.084Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246610367Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163342024-02-03T17:29:47.084Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246645907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163352024-02-03T17:29:47.084Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246769348Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163362024-02-03T17:29:47.084Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.24684193Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163372024-02-03T17:29:47.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.24691452Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163382024-02-03T17:29:47.085Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.246998099Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163392024-02-03T17:29:47.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.248311759Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163402024-02-03T17:29:47.085Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.248436146Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163412024-02-03T17:29:47.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.248516538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163422024-02-03T17:29:47.085Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.248559945Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163432024-02-03T17:29:47.085Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.249330393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163442024-02-03T17:29:47.085Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.249470114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163452024-02-03T17:29:47.085Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.249710894Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163462024-02-03T17:29:47.085Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.249764062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163472024-02-03T17:29:47.085Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.250052977Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163482024-02-03T17:29:47.085Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.250118449Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163492024-02-03T17:29:47.085Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.25014588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163502024-02-03T17:29:47.085Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.250625665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163512024-02-03T17:29:47.085Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.250684883Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163522024-02-03T17:29:47.085Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.250711218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163532024-02-03T17:29:47.085Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.250735003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163542024-02-03T17:29:47.085Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251390006Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163552024-02-03T17:29:47.085Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251465334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163562024-02-03T17:29:47.085Z{"msg":"[0]R flush_numbers[0..12]: [279, 279, 273, 267, 268, 274, 274, 279, 275, 252, 277, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251537914Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163572024-02-03T17:29:47.085Z{"msg":"[0]R generation[0..12]: [151, 151, 148, 144, 145, 148, 148, 151, 149, 137, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251600437Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163582024-02-03T17:29:47.085Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251663447Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163592024-02-03T17:29:47.085Z{"msg":"Wait for a query_work_queue command to finish before sending IO
163602024-02-03T17:29:47.085Z[1]R flush_numbers[0..12]: [279, 279, 273, 267, 268, 274, 274, 279, 275, 252, 277, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.25172745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163612024-02-03T17:29:47.085Z{"msg":"[1]R generation[0..12]: [151, 151, 148, 144, 145, 148, 148, 151, 149, 137, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251791456Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163622024-02-03T17:29:47.086Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251823546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163632024-02-03T17:29:47.086Z{"msg":"[2]R flush_numbers[0..12]: [279, 279, 273, 267, 268, 274, 274, 279, 275, 252, 277, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251896209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163642024-02-03T17:29:47.086Z{"msg":"[2]R generation[0..12]: [151, 151, 148, 144, 145, 148, 148, 151, 149, 137, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.251969367Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163652024-02-03T17:29:47.086Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252043496Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163662024-02-03T17:29:47.086Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252109202Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163672024-02-03T17:29:47.086Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252141205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163682024-02-03T17:29:47.086Z{"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252176332Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163692024-02-03T17:29:47.086Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252200268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163702024-02-03T17:29:47.086Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252260443Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163712024-02-03T17:29:47.086Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252286954Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163722024-02-03T17:29:47.086Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.25230882Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163732024-02-03T17:29:47.086Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252371342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163742024-02-03T17:29:47.086Z1743,"client":"2","":"downstairs","session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163752024-02-03T17:29:47.086Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252409451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163762024-02-03T17:29:47.086Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252462812Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163772024-02-03T17:29:47.086Z{"msg":"97a2cfa5-5c49-4d2a-a50b-84f4a5ebbae0 is now active with session: 29ee76a9-e12f-4b07-aa5f-3989890508de","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252492775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163782024-02-03T17:29:47.086Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252546254Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743,"session_id":"29ee76a9-e12f-4b07-aa5f-3989890508de"}
163792024-02-03T17:29:47.086Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:46.252574857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743}
163802024-02-03T17:29:47.086ZLoading write count information from file "/tmp/test_repair_verify.data"
163812024-02-03T17:29:47.086ZRun Repair workload
163822024-02-03T17:29:47.086Z01/30 Write block 459 len 32768 data: 17 15 14 16 16 15 15 12
163832024-02-03T17:29:47.086Z02/30 Read block 316 len 28672
163842024-02-03T17:29:47.090Z03/30 Read block 116 len 28672
163852024-02-03T17:29:47.090Z04/30 Read block 10 len 16384
163862024-02-03T17:29:47.090Z05/30 Write block 362 len 24576 data: 15 17 15 11 8 9
163872024-02-03T17:29:47.090Z06/30 Read block 397 len 4096
163882024-02-03T17:29:47.090Z07/30 Read block 533 len 28672
163892024-02-03T17:29:47.094Z08/30 Read block 577 len 40960
163902024-02-03T17:29:47.097Z09/30 Write block 355 len 36864 data: 14 18 16 16 17 16 17 16 18
163912024-02-03T17:29:47.097Z10/30 Write block 490 len 4096 data: 11
163922024-02-03T17:29:47.097Z11/30 Read block 478 len 28672
163932024-02-03T17:29:47.100Z12/30 Read block 577 len 36864
163942024-02-03T17:29:47.100Z13/30 Write block 106 len 24576 data: 12 13 14 13 13 12
163952024-02-03T17:29:47.103Z14/30 Flush
163962024-02-03T17:29:47.106Z15/30 Write block 9 len 20480 data: 14 18 15 16 13
163972024-02-03T17:29:47.106Z16/30 Read block 76 len 32768
163982024-02-03T17:29:47.109Z17/30 Read block 466 len 16384
163992024-02-03T17:29:47.109Z18/30 Write block 434 len 8192 data: 13 13
164002024-02-03T17:29:47.109Z19/30 Read block 240 len 24576
164012024-02-03T17:29:47.112Z20/30 Read block 7 len 16384
164022024-02-03T17:29:47.112Z21/30 Read block 262 len 36864
164032024-02-03T17:29:47.115Z22/30 Read block 589 len 8192
164042024-02-03T17:29:47.115Z23/30 Read block 225 len 16384
164052024-02-03T17:29:47.118Z24/30 Write block 390 len 8192 data: 12 12
164062024-02-03T17:29:47.118Z25/30 Write block 61 len 16384 data: 12 10 13 13
164072024-02-03T17:29:47.118Z26/30 Read block 189 len 32768
164082024-02-03T17:29:47.121Z27/30 Write block 455 len 32768 data: 9 14 16 17 18 16 15 17
164092024-02-03T17:29:47.121Z28/30 Read block 308 len 4096
164102024-02-03T17:29:47.125Z29/30 Read block 143 len 12288
164112024-02-03T17:29:47.125Z30/30 Write block 109 len 8192 data: 14 14
164122024-02-03T17:29:47.125Z----------------------------------------------------------------
164132024-02-03T17:29:47.125Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
164142024-02-03T17:29:47.125ZStates: Active Active Active
164152024-02-03T17:29:47.125ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164162024-02-03T17:29:47.125Z 1 Acked 1000 Write 8 Done Done Done false
164172024-02-03T17:29:47.125Z 2 Acked 1001 Read 7 Done Done Done false
164182024-02-03T17:29:47.125Z 3 Acked 1002 Read 7 Done Done Done false
164192024-02-03T17:29:47.125Z 4 Acked 1003 Read 4 Done Done Done false
164202024-02-03T17:29:47.125Z 5 Acked 1004 Write 6 Done Done Done false
164212024-02-03T17:29:47.125Z 6 Acked 1005 Read 1 Done Done Done false
164222024-02-03T17:29:47.126Z 7 Acked 1006 Read 7 Done Done Done false
164232024-02-03T17:29:47.126Z 8 Acked 1007 Read 10 Done Done Done false
164242024-02-03T17:29:47.126Z 9 Acked 1008 Write 9 Done Done Done false
164252024-02-03T17:29:47.126Z 10 Acked 1009 Write 1 Done Done Done false
164262024-02-03T17:29:47.126Z 11 Acked 1010 Read 7 Done Done Done false
164272024-02-03T17:29:47.126Z 12 Acked 1011 Read 9 Done Done Done false
164282024-02-03T17:29:47.126Z 13 Acked 1012 Write 6 Done Done Sent false
164292024-02-03T17:29:47.126Z 14 Acked 1013 Flush 0 Done Done Sent false
164302024-02-03T17:29:47.126Z 15 Acked 1014 Write 5 Done Done Sent false
164312024-02-03T17:29:47.126Z 16 Acked 1015 Read 8 Done Done Sent false
164322024-02-03T17:29:47.126Z 17 Acked 1016 Read 4 Done Done Sent false
164332024-02-03T17:29:47.126Z 18 Acked 1017 Write 2 Done Done Sent false
164342024-02-03T17:29:47.126Z 19 Acked 1018 Read 6 Done Done Sent false
164352024-02-03T17:29:47.126Z 20 Acked 1019 Read 4 Done Done Sent false
164362024-02-03T17:29:47.126Z 21 Acked 1020 Read 9 Done Done Sent false
164372024-02-03T17:29:47.126Z 22 Acked 1021 Read 2 Done Done Sent false
164382024-02-03T17:29:47.126Z 23 Acked 1022 Read 4 Done Done Sent false
164392024-02-03T17:29:47.126Z 24 Acked 1023 Write 2 Done Done Sent false
164402024-02-03T17:29:47.126Z 25 Acked 1024 Write 4 Done Done Sent false
164412024-02-03T17:29:47.126Z 26 Acked 1025 Read 8 Done Done Sent false
164422024-02-03T17:29:47.126Z 27 Acked 1026 Write 8 Done Done Sent false
164432024-02-03T17:29:47.126Z 28 Acked 1027 Read 1 Done Done Sent false
164442024-02-03T17:29:47.126Z 29 Acked 1028 Read 3 Done Done Sent false
164452024-02-03T17:29:47.126Z 30 Acked 1029 Write 2 Sent Sent Sent false
164462024-02-03T17:29:47.126Z STATES DS:0 DS:1 DS:2 TOTAL
164472024-02-03T17:29:47.127Z New 0 0 0 0
164482024-02-03T17:29:47.127Z Sent 1 1 18 20
164492024-02-03T17:29:47.127Z Done 29 29 12 70
164502024-02-03T17:29:47.127Z Skipped 0 0 0 0
164512024-02-03T17:29:47.127Z Error 0 0 0 0
164522024-02-03T17:29:47.127ZLast Flush: 1013 1013 0
164532024-02-03T17:29:47.127ZDownstairs last five completed:
164542024-02-03T17:29:47.127ZUpstairs last five completed: 30 29 28 27 26
164552024-02-03T17:29:47.127Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:46.298703397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1743}
164562024-02-03T17:29:47.130ZWrote out file "/tmp/test_repair_verify.data"
164572024-02-03T17:29:47.133Z
164582024-02-03T17:29:47.140ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164592024-02-03T17:29:47.170ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164602024-02-03T17:29:47.170Z 0 000-019 151 151 151 279 279 279 T T F <---
164612024-02-03T17:29:47.173Z 1 020-039 151 151 151 279 279 279 F F F
164622024-02-03T17:29:47.174Z 2 040-059 148 148 148 273 273 273 F F F
164632024-02-03T17:29:47.174Z 3 060-079 144 144 144 267 267 267 T T F <---
164642024-02-03T17:29:47.174Z 4 080-099 145 145 145 268 268 268 F F F
164652024-02-03T17:29:47.174Z 5 100-119 152 152 148 280 280 274 T T F <---
164662024-02-03T17:29:47.174Z 6 120-139 148 148 148 274 274 274 F F F
164672024-02-03T17:29:47.174Z 7 140-159 151 151 151 279 279 279 F F F
164682024-02-03T17:29:47.174Z 8 160-179 149 149 149 275 275 275 F F F
164692024-02-03T17:29:47.174Z 9 180-199 137 137 137 252 252 252 F F F
164702024-02-03T17:29:47.174Z 10 200-219 150 150 150 277 277 277 F F F
164712024-02-03T17:29:47.174Z 11 220-239 147 147 147 272 272 272 F F F
164722024-02-03T17:29:47.174Z 12 240-259 150 150 150 277 277 277 F F F
164732024-02-03T17:29:47.174Z 13 260-279 148 148 148 273 273 273 F F F
164742024-02-03T17:29:47.174Z 14 280-299 146 146 146 271 271 271 F F F
164752024-02-03T17:29:47.174Z 15 300-319 151 151 151 279 279 279 F F F
164762024-02-03T17:29:47.174Z 16 320-339 140 140 140 259 259 259 F F F
164772024-02-03T17:29:47.174Z 17 340-359 152 152 149 280 280 275 F F T <---
164782024-02-03T17:29:47.174Z 18 360-379 152 152 148 280 280 274 F F T <---
164792024-02-03T17:29:47.174Z 19 380-399 148 148 148 273 273 273 T T F <---
164802024-02-03T17:29:47.174Z 20 400-419 145 145 145 268 268 268 F F F
164812024-02-03T17:29:47.174Z 21 420-439 151 151 151 279 279 279 T T F <---
164822024-02-03T17:29:47.174Z 22 440-459 152 152 150 280 280 278 T T T <---
164832024-02-03T17:29:47.174Z 23 460-479 152 152 150 280 280 278 T T T <---
164842024-02-03T17:29:47.174Z 24 480-499 152 152 150 280 280 278 F F T <---
164852024-02-03T17:29:47.175Z 25 500-519 149 149 149 275 275 275 F F F
164862024-02-03T17:29:47.175Z 26 520-539 149 149 149 275 275 275 F F F
164872024-02-03T17:29:47.175Z 27 540-559 144 144 144 266 266 266 F F F
164882024-02-03T17:29:47.175Z 28 560-579 150 150 150 278 278 278 F F F
164892024-02-03T17:29:47.175Z 29 580-599 150 150 150 278 278 278 F F F
164902024-02-03T17:29:47.175ZMax gen: 152, Max flush: 280
164912024-02-03T17:29:47.175ZError: Difference in extent metadata found!
164922024-02-03T17:29:47.178ZA Difference in extent metadata is expected here
164932024-02-03T17:29:47.178ZOn loop 76
164942024-02-03T17:29:47.179Z
164952024-02-03T17:29:47.181ZVerifying data now
164962024-02-03T17:29:53.119ZNo change after verify
164972024-02-03T17:29:53.119ZLoop: 76 Downstairs dump after verify (and repair):
164982024-02-03T17:29:53.146ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164992024-02-03T17:29:53.146Z 0 000-019 153 153 153 281 281 281 F F F
165002024-02-03T17:29:53.154Z 1 020-039 151 151 151 279 279 279 F F F
165012024-02-03T17:29:53.154Z 2 040-059 148 148 148 273 273 273 F F F
165022024-02-03T17:29:53.154Z 3 060-079 153 153 153 281 281 281 F F F
165032024-02-03T17:29:53.154Z 4 080-099 145 145 145 268 268 268 F F F
165042024-02-03T17:29:53.154Z 5 100-119 153 153 153 281 281 281 F F F
165052024-02-03T17:29:53.154Z 6 120-139 148 148 148 274 274 274 F F F
165062024-02-03T17:29:53.154Z 7 140-159 151 151 151 279 279 279 F F F
165072024-02-03T17:29:53.154Z 8 160-179 149 149 149 275 275 275 F F F
165082024-02-03T17:29:53.154Z 9 180-199 137 137 137 252 252 252 F F F
165092024-02-03T17:29:53.154Z 10 200-219 150 150 150 277 277 277 F F F
165102024-02-03T17:29:53.154Z 11 220-239 147 147 147 272 272 272 F F F
165112024-02-03T17:29:53.155Z 12 240-259 150 150 150 277 277 277 F F F
165122024-02-03T17:29:53.155Z 13 260-279 148 148 148 273 273 273 F F F
165132024-02-03T17:29:53.155Z 14 280-299 146 146 146 271 271 271 F F F
165142024-02-03T17:29:53.155Z 15 300-319 151 151 151 279 279 279 F F F
165152024-02-03T17:29:53.155Z 16 320-339 140 140 140 259 259 259 F F F
165162024-02-03T17:29:53.155Z 17 340-359 152 152 152 280 280 280 F F F
165172024-02-03T17:29:53.155Z 18 360-379 152 152 152 280 280 280 F F F
165182024-02-03T17:29:53.155Z 19 380-399 153 153 153 281 281 281 F F F
165192024-02-03T17:29:53.155Z 20 400-419 145 145 145 268 268 268 F F F
165202024-02-03T17:29:53.155Z 21 420-439 153 153 153 281 281 281 F F F
165212024-02-03T17:29:53.155Z 22 440-459 153 153 153 281 281 281 F F F
165222024-02-03T17:29:53.155Z 23 460-479 153 153 153 281 281 281 F F F
165232024-02-03T17:29:53.156Z 24 480-499 152 152 152 280 280 280 F F F
165242024-02-03T17:29:53.156Z 25 500-519 149 149 149 275 275 275 F F F
165252024-02-03T17:29:53.156Z 26 520-539 149 149 149 275 275 275 F F F
165262024-02-03T17:29:53.156Z 27 540-559 144 144 144 266 266 266 F F F
165272024-02-03T17:29:53.156Z 28 560-579 150 150 150 278 278 278 F F F
165282024-02-03T17:29:53.156Z 29 580-599 150 150 150 278 278 278 F F F
165292024-02-03T17:29:53.156ZMax gen: 153, Max flush: 281
165302024-02-03T17:29:53.156Z
165312024-02-03T17:29:53.156ZBegin loop 77
165322024-02-03T17:29:53.156ZDownstairs to restart: 1
165332024-02-03T17:29:53.185Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.355635009Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751}
165342024-02-03T17:29:53.185Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356196009Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751}
165352024-02-03T17:29:53.185Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356228705Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751}
165362024-02-03T17:29:53.207Z{"msg":"Crucible c295a166-6172-481c-bc85-f1fed7c519ab has session id: 3cbd6787-0a84-42da-9ace-fc4a4debe60b","v":0,"name":"crucible","level":30Crucible runtime is spawned
165372024-02-03T17:29:53.207Z,"time":"2024-02-03T17:29:52.356341211Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165382024-02-03T17:29:53.207Z{"msg":"Upstairs opts: Upstairs UUID: c295a166-6172-481c-bc85-f1fed7c519ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356624072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165392024-02-03T17:29:53.207Z{"msg":"Crucible stats registered with UUID: c295a166-6172-481c-bc85-f1fed7c519ab","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356683598Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165402024-02-03T17:29:53.207Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356727199Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751}
165412024-02-03T17:29:53.207Z{"msg":"c295a166-6172-481c-bc85-f1fed7c519ab active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356764096Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165422024-02-03T17:29:53.207Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356825977Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165432024-02-03T17:29:53.207Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356871157Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165442024-02-03T17:29:53.207Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356945745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165452024-02-03T17:29:53.207Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.356991488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165462024-02-03T17:29:53.208Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.357073902Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165472024-02-03T17:29:53.208Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.357149782Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165482024-02-03T17:29:53.208Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.357192826Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165492024-02-03T17:29:53.208Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.357409315Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165502024-02-03T17:29:53.208Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.357456761Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165512024-02-03T17:29:53.208Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.357482649Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165522024-02-03T17:29:53.208Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.357537405Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165532024-02-03T17:29:53.208Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.357569416Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165542024-02-03T17:29:53.208Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.358919127Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165552024-02-03T17:29:53.208Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.359002648Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165562024-02-03T17:29:53.208Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.359028712Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165572024-02-03T17:29:53.208Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.359260448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165582024-02-03T17:29:53.208Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.359729301Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165592024-02-03T17:29:53.209Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.359886383Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165602024-02-03T17:29:53.209Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.3601031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165612024-02-03T17:29:53.209Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.360189668Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165622024-02-03T17:29:53.209Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.360782835Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165632024-02-03T17:29:53.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361167049Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165642024-02-03T17:29:53.209Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361255793Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165652024-02-03T17:29:53.209Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361330579Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165662024-02-03T17:29:53.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361398641Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165672024-02-03T17:29:53.209Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361471474Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165682024-02-03T17:29:53.209Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361541368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165692024-02-03T17:29:53.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361646034Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165702024-02-03T17:29:53.209Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361709061Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165712024-02-03T17:29:53.209Z{"msg":"[0]R flush_numbers[0..12]: [281, 279, 273, 281, 268, 281, 274, 279, 275, 252, 277, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361737123Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165722024-02-03T17:29:53.209Z{"msg":"[0]R generation[0..12]: [153, 151, 148, 153, 145, 153, 148, 151, 149, 137, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361761784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165732024-02-03T17:29:53.210Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361785934Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165742024-02-03T17:29:53.210Z{"msg":"[1]R flush_numbers[0..12]: [281, 279, 273, 281, 268, 281, 274, 279, 275, 252, 277, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361810564Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165752024-02-03T17:29:53.210Z{"msg":"[1]R generation[0..12]: [153, 151, 148, 153, 145, 153, 148, 151, 149, 137, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361853053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165762024-02-03T17:29:53.210Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361878477Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165772024-02-03T17:29:53.210Z{"msg":"[2]R flush_numbers[0..12]: [281, 279, 273, 281, 268, 281, 274, 279, 275, 252, 277, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361902521Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165782024-02-03T17:29:53.210Z{"msg":"[2]R generation[0..12]: [153, 151, 148, 153, 145, 153, 148, 151, 149, 137, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361943946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165792024-02-03T17:29:53.210Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361969259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165802024-02-03T17:29:53.210Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.361991225Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165812024-02-03T17:29:53.210Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.36203191Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165822024-02-03T17:29:53.210ZWait for a query_work_queue command to finish before sending IO
165832024-02-03T17:29:53.210Z{"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362061049Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165842024-02-03T17:29:53.210Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362083019Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165852024-02-03T17:29:53.211Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362123467Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165862024-02-03T17:29:53.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362145705Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165872024-02-03T17:29:53.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362167441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165882024-02-03T17:29:53.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362213694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165892024-02-03T17:29:53.211Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362237505Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165902024-02-03T17:29:53.211Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362258269Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165912024-02-03T17:29:53.211Z{"msg":"c295a166-6172-481c-bc85-f1fed7c519ab is now active with session: 3cbd6787-0a84-42da-9ace-fc4a4debe60b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362302265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165922024-02-03T17:29:53.211Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.362324262Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751,"session_id":"3cbd6787-0a84-42da-9ace-fc4a4debe60b"}
165932024-02-03T17:29:53.211Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:52.36234449Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751}
165942024-02-03T17:29:53.211ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165952024-02-03T17:29:53.211ZLoading write count information from file "/tmp/test_repair_verify.data"
165962024-02-03T17:29:53.211ZRun Repair workload
165972024-02-03T17:29:53.211Z01/30 Write block 459 len 16384 data: 19 17 16 18
165982024-02-03T17:29:53.219Z02/30 Read block 97 len 40960
165992024-02-03T17:29:53.219Z03/30 Write block 485 len 40960 data: 8 10 10 10 12 12 12 11 14 13
166002024-02-03T17:29:53.219Z04/30 Read block 251 len 40960
166012024-02-03T17:29:53.220Z05/30 Write block 506 len 4096 data: 24
166022024-02-03T17:29:53.220Z06/30 Write block 334 len 8192 data: 13 11
166032024-02-03T17:29:53.220Z07/30 Write block 123 len 8192 data: 12 9
166042024-02-03T17:29:53.220Z08/30 Write block 592 len 12288 data: 12 11 8
166052024-02-03T17:29:53.220Z09/30 Write block 125 len 8192 data: 10 13
166062024-02-03T17:29:53.220Z10/30 Write block 455 len 40960 data: 10 15 17 18 20 18 17 19 17 16
166072024-02-03T17:29:53.220Z11/30 Read block 65 len 20480
166082024-02-03T17:29:53.220Z12/30 Read block 400 len 12288
166092024-02-03T17:29:53.220Z13/30 Write block 197 len 36864 data: 8 9 10 8 7 8 10 9 8
166102024-02-03T17:29:53.220Z14/30 Write block 58 len 28672 data: 14 14 13 13 11 14 14
166112024-02-03T17:29:53.220Z15/30 Read block 25 len 8192
166122024-02-03T17:29:53.220Z16/30 Flush
166132024-02-03T17:29:53.226Z17/30 Write block 245 len 28672 data: 6 7 9 10 11 16 11
166142024-02-03T17:29:53.226Z18/30 Read block 363 len 32768
166152024-02-03T17:29:53.226Z19/30 Write block 471 len 24576 data: 8 7 6 7 6 7
166162024-02-03T17:29:53.226Z20/30 Read block 483 len 8192
166172024-02-03T17:29:53.226Z21/30 Write block 113 len 24576 data: 15 17 16 17 16 15
166182024-02-03T17:29:53.226Z22/30 Read block 6 len 20480
166192024-02-03T17:29:53.226Z23/30 Flush
166202024-02-03T17:29:53.236Z24/30 Write block 553 len 8192 data: 12 11
166212024-02-03T17:29:53.250Z25/30 Write block 282 len 36864 data: 18 16 15 12 11 12 11 10 9
166222024-02-03T17:29:53.250Z26/30 Read block 71 len 12288
166232024-02-03T17:29:53.250Z27/30 Write block 498 len 8192 data: 13 15
166242024-02-03T17:29:53.250Z28/30 Write block 58 len 36864 data: 15 15 14 14 12 15 15 10 9
166252024-02-03T17:29:53.250Z29/30 Write block 395 len 36864 data: 12 13 10 9 10 11 9 9 12
166262024-02-03T17:29:53.251Z30/30 Write block 474 len 20480 data: 8 7 8 10 12
166272024-02-03T17:29:53.251Z----------------------------------------------------------------
166282024-02-03T17:29:53.251Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166292024-02-03T17:29:53.251ZStates: Active Active Active
166302024-02-03T17:29:53.251ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166312024-02-03T17:29:53.251Z 1 Acked 1000 Write 4 Done Done Done false
166322024-02-03T17:29:53.251Z 2 Acked 1001 Read 10 Done Sent Done false
166332024-02-03T17:29:53.252Z 3 Acked 1002 Write 10 Done Sent Done false
166342024-02-03T17:29:53.252Z 4 Acked 1003 Read 10 Done Sent Done false
166352024-02-03T17:29:53.252Z 5 Acked 1004 Write 1 Done Sent Done false
166362024-02-03T17:29:53.252Z 6 Acked 1005 Write 2 Done Sent Done false
166372024-02-03T17:29:53.252Z 7 Acked 1006 Write 2 Done Sent Done false
166382024-02-03T17:29:53.252Z 8 Acked 1007 Write 3 Done Sent Done false
166392024-02-03T17:29:53.253Z 9 Acked 1008 Write 2 Done Sent Done false
166402024-02-03T17:29:53.253Z 10 Acked 1009 Write 10 Done Sent Done false
166412024-02-03T17:29:53.253Z 11 Acked 1010 Read 5 Done Sent Done false
166422024-02-03T17:29:53.253Z 12 Acked 1011 Read 3 Done Sent Done false
166432024-02-03T17:29:53.253Z 13 Acked 1012 Write 9 Done Sent Done false
166442024-02-03T17:29:53.253Z 14 Acked 1013 Write 7 Done Sent Done false
166452024-02-03T17:29:53.253Z 15 Acked 1014 Read 2 Done Sent Done false
166462024-02-03T17:29:53.254Z 16 Acked 1015 Flush 0 Done Sent Done false
166472024-02-03T17:29:53.254Z 17 Acked 1016 Write 7 Done Sent Done false
166482024-02-03T17:29:53.254Z 18 Acked 1017 Read 8 Done Sent Done false
166492024-02-03T17:29:53.254Z 19 Acked 1018 Write 6 Done Sent Done false
166502024-02-03T17:29:53.254Z 20 Acked 1019 Read 2 Done Sent Done false
166512024-02-03T17:29:53.255Z 21 Acked 1020 Write 6 Done Sent Done false
166522024-02-03T17:29:53.255Z 22 Acked 1021 Read 5 Done Sent Done false
166532024-02-03T17:29:53.255Z 23 Acked 1022 Flush 0 Done Sent Done false
166542024-02-03T17:29:53.255Z 24 Acked 1023 Write 2 Done Sent Done false
166552024-02-03T17:29:53.256Z 25 Acked 1024 Write 9 Done Sent Done false
166562024-02-03T17:29:53.256Z 26 Acked 1025 Read 3 Done Sent Done false
166572024-02-03T17:29:53.256Z 27 Acked 1026 Write 2 Done Sent Done false
166582024-02-03T17:29:53.256Z 28 Acked 1027 Write 9 Done Sent Done false
166592024-02-03T17:29:53.256Z 29 Acked 1028 Write 9 Sent Sent Sent false
166602024-02-03T17:29:53.256Z 30 Acked 1029 Write 5 Sent Sent Sent false
166612024-02-03T17:29:53.256Z STATES DS:0 DS:1 DS:2 TOTAL
166622024-02-03T17:29:53.257Z New 0 0 0 0
166632024-02-03T17:29:53.257Z Sent 2 29 2 33
166642024-02-03T17:29:53.257Z Done 28 1 28 57
166652024-02-03T17:29:53.257Z Skipped 0 0 0 0
166662024-02-03T17:29:53.258Z Error 0 0 0 0
166672024-02-03T17:29:53.258ZLast Flush: 1022 0 1022
166682024-02-03T17:29:53.258ZDownstairs last five completed:
166692024-02-03T17:29:53.258ZUpstairs last five completed: 30 29 28 27 26
166702024-02-03T17:29:53.258Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:52.412992355Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1751}
166712024-02-03T17:29:53.259ZWrote out file "/tmp/test_repair_verify.data"
166722024-02-03T17:29:53.678Z
166732024-02-03T17:29:53.678ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166742024-02-03T17:29:53.683ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166752024-02-03T17:29:53.683Z 0 000-019 153 153 153 281 281 281 F F F
166762024-02-03T17:29:53.683Z 1 020-039 151 151 151 279 279 279 F F F
166772024-02-03T17:29:53.683Z 2 040-059 154 148 154 282 273 282 T F T <---
166782024-02-03T17:29:53.683Z 3 060-079 154 153 154 282 281 282 T F T <---
166792024-02-03T17:29:53.683Z 4 080-099 145 145 145 268 268 268 F F F
166802024-02-03T17:29:53.683Z 5 100-119 154 153 154 283 281 283 F F F <---
166812024-02-03T17:29:53.683Z 6 120-139 154 148 154 282 274 282 F F F <---
166822024-02-03T17:29:53.683Z 7 140-159 151 151 151 279 279 279 F F F
166832024-02-03T17:29:53.683Z 8 160-179 149 149 149 275 275 275 F F F
166842024-02-03T17:29:53.683Z 9 180-199 154 137 154 282 252 282 F F F <---
166852024-02-03T17:29:53.683Z 10 200-219 154 150 154 282 277 282 F F F <---
166862024-02-03T17:29:53.684Z 11 220-239 147 147 147 272 272 272 F F F
166872024-02-03T17:29:53.684Z 12 240-259 154 150 154 283 277 283 F F F <---
166882024-02-03T17:29:53.684Z 13 260-279 148 148 148 273 273 273 F F F
166892024-02-03T17:29:53.684Z 14 280-299 146 146 146 271 271 271 T F T <---
166902024-02-03T17:29:53.684Z 15 300-319 151 151 151 279 279 279 F F F
166912024-02-03T17:29:53.684Z 16 320-339 154 140 154 282 259 282 F F F <---
166922024-02-03T17:29:53.684Z 17 340-359 152 152 152 280 280 280 F F F
166932024-02-03T17:29:53.684Z 18 360-379 152 152 152 280 280 280 F F F
166942024-02-03T17:29:53.684Z 19 380-399 153 153 153 281 281 281 T F T <---
166952024-02-03T17:29:53.684Z 20 400-419 145 145 145 268 268 268 T F T <---
166962024-02-03T17:29:53.684Z 21 420-439 153 153 153 281 281 281 F F F
166972024-02-03T17:29:53.684Z 22 440-459 154 153 154 282 281 282 F T F <---
166982024-02-03T17:29:53.684Z 23 460-479 154 153 154 283 281 283 T T T <---
166992024-02-03T17:29:53.685Z 24 480-499 154 152 154 282 280 282 T F T <---
167002024-02-03T17:29:53.685Z 25 500-519 154 149 154 282 275 282 F F F <---
167012024-02-03T17:29:53.685Z 26 520-539 149 149 149 275 275 275 F F F
167022024-02-03T17:29:53.685Z 27 540-559 144 144 144 266 266 266 T F T <---
167032024-02-03T17:29:53.685Z 28 560-579 150 150 150 278 278 278 F F F
167042024-02-03T17:29:53.685Z 29 580-599 154 150 154 282 278 282 F F F <---
167052024-02-03T17:29:53.685ZMax gen: 154, Max flush: 283
167062024-02-03T17:29:53.685ZError: Difference in extent metadata found!
167072024-02-03T17:29:53.685ZA Difference in extent metadata is expected here
167082024-02-03T17:29:53.685ZOn loop 77
167092024-02-03T17:29:53.685Z
167102024-02-03T17:29:53.685ZVerifying data now
167112024-02-03T17:29:59.740ZNo change after verify
167122024-02-03T17:29:59.740ZLoop: 77 Downstairs dump after verify (and repair):
167132024-02-03T17:29:59.764ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167142024-02-03T17:29:59.764Z 0 000-019 153 153 153 281 281 281 F F F
167152024-02-03T17:29:59.768Z 1 020-039 151 151 151 279 279 279 F F F
167162024-02-03T17:29:59.768Z 2 040-059 155 155 155 284 284 284 F F F
167172024-02-03T17:29:59.768Z 3 060-079 155 155 155 284 284 284 F F F
167182024-02-03T17:29:59.768Z 4 080-099 145 145 145 268 268 268 F F F
167192024-02-03T17:29:59.768Z 5 100-119 154 154 154 283 283 283 F F F
167202024-02-03T17:29:59.768Z 6 120-139 154 154 154 282 282 282 F F F
167212024-02-03T17:29:59.768Z 7 140-159 151 151 151 279 279 279 F F F
167222024-02-03T17:29:59.768Z 8 160-179 149 149 149 275 275 275 F F F
167232024-02-03T17:29:59.769Z 9 180-199 154 154 154 282 282 282 F F F
167242024-02-03T17:29:59.769Z 10 200-219 154 154 154 282 282 282 F F F
167252024-02-03T17:29:59.769Z 11 220-239 147 147 147 272 272 272 F F F
167262024-02-03T17:29:59.769Z 12 240-259 154 154 154 283 283 283 F F F
167272024-02-03T17:29:59.769Z 13 260-279 148 148 148 273 273 273 F F F
167282024-02-03T17:29:59.769Z 14 280-299 155 155 155 284 284 284 F F F
167292024-02-03T17:29:59.769Z 15 300-319 151 151 151 279 279 279 F F F
167302024-02-03T17:29:59.769Z 16 320-339 154 154 154 282 282 282 F F F
167312024-02-03T17:29:59.769Z 17 340-359 152 152 152 280 280 280 F F F
167322024-02-03T17:29:59.769Z 18 360-379 152 152 152 280 280 280 F F F
167332024-02-03T17:29:59.770Z 19 380-399 155 155 155 284 284 284 F F F
167342024-02-03T17:29:59.770Z 20 400-419 155 155 155 284 284 284 F F F
167352024-02-03T17:29:59.770Z 21 420-439 153 153 153 281 281 281 F F F
167362024-02-03T17:29:59.770Z 22 440-459 154 154 154 282 282 282 F F F
167372024-02-03T17:29:59.770Z 23 460-479 155 155 155 284 284 284 F F F
167382024-02-03T17:29:59.770Z 24 480-499 155 155 155 284 284 284 F F F
167392024-02-03T17:29:59.770Z 25 500-519 154 154 154 282 282 282 F F F
167402024-02-03T17:29:59.770Z 26 520-539 149 149 149 275 275 275 F F F
167412024-02-03T17:29:59.770Z 27 540-559 155 155 155 284 284 284 F F F
167422024-02-03T17:29:59.770Z 28 560-579 150 150 150 278 278 278 F F F
167432024-02-03T17:29:59.770Z 29 580-599 154 154 154 282 282 282 F F F
167442024-02-03T17:29:59.771ZMax gen: 155, Max flush: 284
167452024-02-03T17:29:59.780Z
167462024-02-03T17:29:59.780ZBegin loop 78
167472024-02-03T17:29:59.780ZDownstairs to restart: 2
167482024-02-03T17:29:59.809Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.976685937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759}
167492024-02-03T17:29:59.809ZCrucible runtime is spawned
167502024-02-03T17:29:59.814Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977264247Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759}
167512024-02-03T17:29:59.814Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977357123Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759}
167522024-02-03T17:29:59.814Z{"msg":"Crucible 47f2ee88-11da-469b-96ca-285a46bec2a9 has session id: c7597bc2-2615-4c54-a9c4-91a0191fa40b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977383745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167532024-02-03T17:29:59.815Z{"msg":"Upstairs opts: Upstairs UUID: 47f2ee88-11da-469b-96ca-285a46bec2a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977424893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167542024-02-03T17:29:59.815Z{"msg":"Crucible stats registered with UUID: 47f2ee88-11da-469b-96ca-285a46bec2a9","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.97747263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167552024-02-03T17:29:59.815Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977496396Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759}
167562024-02-03T17:29:59.815Z{"msg":"47f2ee88-11da-469b-96ca-285a46bec2a9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977534436Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167572024-02-03T17:29:59.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977556955Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167582024-02-03T17:29:59.815Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.97761867Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167592024-02-03T17:29:59.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.97766138Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167602024-02-03T17:29:59.815Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977731459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167612024-02-03T17:29:59.815Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977903692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167622024-02-03T17:29:59.815Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.977991913Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167632024-02-03T17:29:59.815Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.97803858Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167642024-02-03T17:29:59.815Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.978086692Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167652024-02-03T17:29:59.815Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.978128716Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167662024-02-03T17:29:59.815Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.97822544Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167672024-02-03T17:29:59.815Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.978272338Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167682024-02-03T17:29:59.815Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.978365515Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167692024-02-03T17:29:59.815Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.979696939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167702024-02-03T17:29:59.815Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.979753946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167712024-02-03T17:29:59.815Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.979795917Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167722024-02-03T17:29:59.815Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.979883246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167732024-02-03T17:29:59.815Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.980159727Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167742024-02-03T17:29:59.815Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.980232694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167752024-02-03T17:29:59.815Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.980770675Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167762024-02-03T17:29:59.815Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.981104151Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167772024-02-03T17:29:59.816Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.981463001Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167782024-02-03T17:29:59.816Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.981574557Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167792024-02-03T17:29:59.816Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.981663519Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167802024-02-03T17:29:59.816Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.981708481Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167812024-02-03T17:29:59.816Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982016669Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167822024-02-03T17:29:59.816Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982095341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167832024-02-03T17:29:59.816Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982123592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167842024-02-03T17:29:59.816Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982340031Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167852024-02-03T17:29:59.816Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982422449Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167862024-02-03T17:29:59.816Z{"msg":"[0]R flush_numbers[0..12]: [281, 279, 284, 284, 268, 283, 282, 279, 275, 282, 282, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982495347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167872024-02-03T17:29:59.816Z{"msg":"[0]R generation[0..12]: [153, 151, 155, 155, 145, 154, 154, 151, 149, 154, 154, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982569005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167882024-02-03T17:29:59.816Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982640778Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167892024-02-03T17:29:59.816Z{"msg":"[1]R flush_numbers[0..12]: [281, 279, 284, 284, 268, 283, 282, 279, 275, 282, 282, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982715595Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167902024-02-03T17:29:59.816Z{"msg":"Wait for a query_work_queue command to finish before sending IO
167912024-02-03T17:29:59.816Z[1]R generation[0..12]: [153, 151, 155, 155, 145, 154, 154, 151, 149, 154, 154, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982880245Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167922024-02-03T17:29:59.816Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.982935681Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167932024-02-03T17:29:59.816Z{"msg":"[2]R flush_numbers[0..12]: [281, 279, 284, 284, 268, 283, 282, 279, 275, 282, 282, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983027723Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167942024-02-03T17:29:59.816Z{"msg":"[2]R generation[0..12]: [153, 151, 155, 155, 145, 154, 154, 151, 149, 154, 154, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983087713Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167952024-02-03T17:29:59.816Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983157938Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167962024-02-03T17:29:59.816Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983186957Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167972024-02-03T17:29:59.816Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983251779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167982024-02-03T17:29:59.816Z{"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.98331564Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
167992024-02-03T17:29:59.816Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983356134Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168002024-02-03T17:29:59.816Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.98340337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168012024-02-03T17:29:59.816Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983429466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168022024-02-03T17:29:59.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983495695Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
168032024-02-03T17:29:59.817Zhostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168042024-02-03T17:29:59.817Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983529461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168052024-02-03T17:29:59.817Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983588858Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168062024-02-03T17:29:59.817Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983615783Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168072024-02-03T17:29:59.817Z{"msg":"47f2ee88-11da-469b-96ca-285a46bec2a9 is now active with session: c7597bc2-2615-4c54-a9c4-91a0191fa40b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983658234Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168082024-02-03T17:29:59.817Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983680934Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759,"session_id":"c7597bc2-2615-4c54-a9c4-91a0191fa40b"}
168092024-02-03T17:29:59.817Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:29:58.983720364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759}
168102024-02-03T17:29:59.817ZLoading write count information from file "/tmp/test_repair_verify.data"
168112024-02-03T17:29:59.817ZRun Repair workload
168122024-02-03T17:29:59.817Z01/30 Write block 204 len 4096 data: 10
168132024-02-03T17:29:59.817Z02/30 Read block 70 len 40960
168142024-02-03T17:29:59.824Z03/30 Write block 365 len 24576 data: 12 9 10 9 8 7
168152024-02-03T17:29:59.824Z04/30 Write block 418 len 40960 data: 12 13 16 14 13 14 13 11 10 10
168162024-02-03T17:29:59.824Z05/30 Read block 152 len 28672
168172024-02-03T17:29:59.824Z06/30 Read block 380 len 8192
168182024-02-03T17:29:59.824Z07/30 Read block 463 len 40960
168192024-02-03T17:29:59.880Z08/30 Read block 41 len 12288
168202024-02-03T17:29:59.880Z09/30 Read block 228 len 8192
168212024-02-03T17:29:59.880Z10/30 Read block 118 len 12288
168222024-02-03T17:29:59.880Z11/30 Write block 546 len 20480 data: 11 11 15 16 14
168232024-02-03T17:29:59.880Z12/30 Write block 593 len 16384 data: 12 9 9 9
168242024-02-03T17:29:59.880Z13/30 Write block 496 len 24576 data: 12 11 14 16 16 18
168252024-02-03T17:29:59.880Z14/30 Read block 556 len 12288
168262024-02-03T17:29:59.880Z15/30 Write block 384 len 4096 data: 10
168272024-02-03T17:29:59.880Z16/30 Read block 475 len 28672
168282024-02-03T17:29:59.880Z17/30 Write block 548 len 8192 data: 16 17
168292024-02-03T17:29:59.880Z18/30 Read block 549 len 4096
168302024-02-03T17:29:59.880Z19/30 Write block 464 len 32768 data: 17 16 13 13 9 9 9 9
168312024-02-03T17:29:59.880Z20/30 Write block 556 len 24576 data: 12 11 12 13 9 7
168322024-02-03T17:29:59.880Z21/30 Write block 210 len 16384 data: 12 10 9 9
168332024-02-03T17:29:59.880Z22/30 Flush
168342024-02-03T17:29:59.880Z23/30 Write block 330 len 8192 data: 11 12
168352024-02-03T17:29:59.880Z24/30 Flush
168362024-02-03T17:29:59.881Z25/30 Write block 74 len 36864 data: 9 10 11 11 10 9 9 8 10
168372024-02-03T17:29:59.881Z26/30 Flush
168382024-02-03T17:29:59.881Z27/30 Write block 381 len 8192 data: 11 10
168392024-02-03T17:29:59.881Z28/30 Write block 62 len 32768 data: 13 16 16 11 10 8 6 7
168402024-02-03T17:29:59.881Z29/30 Write block 117 len 12288 data: 17 16 15
168412024-02-03T17:29:59.881Z30/30 Read block 543 len 32768
168422024-02-03T17:29:59.881Z----------------------------------------------------------------
168432024-02-03T17:29:59.881Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
168442024-02-03T17:29:59.881ZStates: Active Active Active
168452024-02-03T17:29:59.881ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168462024-02-03T17:29:59.881Z 1 Acked 1000 Write 1 Done Done Done false
168472024-02-03T17:29:59.881Z 2 Acked 1001 Read 10 Done Done Done false
168482024-02-03T17:29:59.881Z 3 Acked 1002 Write 6 Done Done Done false
168492024-02-03T17:29:59.881Z 4 Acked 1003 Write 10 Done Done Done false
168502024-02-03T17:29:59.881Z 5 Acked 1004 Read 7 Done Done Sent false
168512024-02-03T17:29:59.881Z 6 Acked 1005 Read 2 Done Done Sent false
168522024-02-03T17:29:59.881Z 7 Acked 1006 Read 10 Done Done Sent false
168532024-02-03T17:29:59.881Z 8 Acked 1007 Read 3 Done Done Sent false
168542024-02-03T17:29:59.881Z 9 Acked 1008 Read 2 Done Done Sent false
168552024-02-03T17:29:59.881Z 10 Acked 1009 Read 3 Done Done Sent false
168562024-02-03T17:29:59.881Z 11 Acked 1010 Write 5 Done Done Sent false
168572024-02-03T17:29:59.881Z 12 Acked 1011 Write 4 Done Done Sent false
168582024-02-03T17:29:59.881Z 13 Acked 1012 Write 6 Done Done Sent false
168592024-02-03T17:29:59.881Z 14 Acked 1013 Read 3 Done Done Sent false
168602024-02-03T17:29:59.882Z 15 Acked 1014 Write 1 Done Done Sent false
168612024-02-03T17:29:59.882Z 16 Acked 1015 Read 7 Done Done Sent false
168622024-02-03T17:29:59.882Z 17 Acked 1016 Write 2 Done Done Sent false
168632024-02-03T17:29:59.882Z 18 Acked 1017 Read 1 Done Done Sent false
168642024-02-03T17:29:59.882Z 19 Acked 1018 Write 8 Done Done Sent false
168652024-02-03T17:29:59.882Z 20 Acked 1019 Write 6 Done Done Sent false
168662024-02-03T17:29:59.882Z 21 Acked 1020 Write 4 Done Done Sent false
168672024-02-03T17:29:59.882Z 22 Acked 1021 Flush 0 Done Done Sent false
168682024-02-03T17:29:59.882Z 23 Acked 1022 Write 2 Done Done Sent false
168692024-02-03T17:29:59.882Z 24 Acked 1023 Flush 0 Done Done Sent false
168702024-02-03T17:29:59.882Z 25 Acked 1024 Write 9 Done Done Sent false
168712024-02-03T17:29:59.882Z 26 Acked 1025 Flush 0 Done Done Sent false
168722024-02-03T17:29:59.882Z 27 Acked 1026 Write 2 Done Done Sent false
168732024-02-03T17:29:59.882Z 28 Acked 1027 Write 8 Done Done Sent false
168742024-02-03T17:29:59.882Z 29 Acked 1028 Write 3 Done Done Sent false
168752024-02-03T17:29:59.882Z 30 Acked 1029 Read 8 Sent Done Sent false
168762024-02-03T17:29:59.882Z STATES DS:0 DS:1 DS:2 TOTAL
168772024-02-03T17:29:59.882Z New 0 0 0 0
168782024-02-03T17:29:59.882Z Sent 1 0 26 27
168792024-02-03T17:29:59.882Z Done 29 30 4 63
168802024-02-03T17:29:59.882Z Skipped 0 0 0 0
168812024-02-03T17:29:59.882Z Error 0 0 0 0
168822024-02-03T17:29:59.882ZLast Flush: 1025 1025 0
168832024-02-03T17:29:59.882ZDownstairs last five completed:
168842024-02-03T17:29:59.883ZUpstairs last five completed: 30 29 28 27 26
168852024-02-03T17:29:59.883Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:29:59.031432282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1759}
168862024-02-03T17:29:59.883ZWrote out file "/tmp/test_repair_verify.data"
168872024-02-03T17:29:59.883Z
168882024-02-03T17:29:59.883ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168892024-02-03T17:29:59.899ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168902024-02-03T17:29:59.899Z 0 000-019 153 153 153 281 281 281 F F F
168912024-02-03T17:29:59.921Z 1 020-039 151 151 151 279 279 279 F F F
168922024-02-03T17:29:59.921Z 2 040-059 155 155 155 284 284 284 F F F
168932024-02-03T17:29:59.921Z 3 060-079 156 156 155 287 287 284 T T F <---
168942024-02-03T17:29:59.921Z 4 080-099 156 156 145 287 287 268 F F F <---
168952024-02-03T17:29:59.921Z 5 100-119 154 154 154 283 283 283 T T F <---
168962024-02-03T17:29:59.921Z 6 120-139 154 154 154 282 282 282 F F F
168972024-02-03T17:29:59.921Z 7 140-159 151 151 151 279 279 279 F F F
168982024-02-03T17:29:59.921Z 8 160-179 149 149 149 275 275 275 F F F
168992024-02-03T17:29:59.921Z 9 180-199 154 154 154 282 282 282 F F F
169002024-02-03T17:29:59.921Z 10 200-219 156 156 154 285 285 282 F F T <---
169012024-02-03T17:29:59.921Z 11 220-239 147 147 147 272 272 272 F F F
169022024-02-03T17:29:59.921Z 12 240-259 154 154 154 283 283 283 F F F
169032024-02-03T17:29:59.922Z 13 260-279 148 148 148 273 273 273 F F F
169042024-02-03T17:29:59.922Z 14 280-299 155 155 155 284 284 284 F F F
169052024-02-03T17:29:59.922Z 15 300-319 151 151 151 279 279 279 F F F
169062024-02-03T17:29:59.922Z 16 320-339 156 156 154 286 286 282 F F F <---
169072024-02-03T17:29:59.922Z 17 340-359 152 152 152 280 280 280 F F F
169082024-02-03T17:29:59.922Z 18 360-379 156 156 152 285 285 280 F F T <---
169092024-02-03T17:29:59.922Z 19 380-399 156 156 155 285 285 284 T T F <---
169102024-02-03T17:29:59.922Z 20 400-419 156 156 155 285 285 284 F F T <---
169112024-02-03T17:29:59.922Z 21 420-439 156 156 153 285 285 281 F F T <---
169122024-02-03T17:29:59.922Z 22 440-459 154 154 154 282 282 282 F F F
169132024-02-03T17:29:59.922Z 23 460-479 156 156 155 285 285 284 F F F <---
169142024-02-03T17:29:59.922Z 24 480-499 156 156 155 285 285 284 F F F <---
169152024-02-03T17:29:59.922Z 25 500-519 156 156 154 285 285 282 F F F <---
169162024-02-03T17:29:59.922Z 26 520-539 149 149 149 275 275 275 F F F
169172024-02-03T17:29:59.922Z 27 540-559 156 156 155 285 285 284 F F F <---
169182024-02-03T17:29:59.922Z 28 560-579 156 156 150 285 285 278 F F F <---
169192024-02-03T17:29:59.922Z 29 580-599 156 156 154 285 285 282 F F F <---
169202024-02-03T17:29:59.922ZMax gen: 156, Max flush: 287
169212024-02-03T17:29:59.922ZError: Difference in extent metadata found!
169222024-02-03T17:29:59.926ZA Difference in extent metadata is expected here
169232024-02-03T17:29:59.926ZOn loop 78
169242024-02-03T17:29:59.926Z
169252024-02-03T17:29:59.926ZVerifying data now
169262024-02-03T17:30:06.209ZNo change after verify
169272024-02-03T17:30:06.209ZLoop: 78 Downstairs dump after verify (and repair):
169282024-02-03T17:30:06.233ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169292024-02-03T17:30:06.233Z 0 000-019 153 153 153 281 281 281 F F F
169302024-02-03T17:30:06.244Z 1 020-039 151 151 151 279 279 279 F F F
169312024-02-03T17:30:06.244Z 2 040-059 155 155 155 284 284 284 F F F
169322024-02-03T17:30:06.244Z 3 060-079 157 157 157 288 288 288 F F F
169332024-02-03T17:30:06.245Z 4 080-099 156 156 156 287 287 287 F F F
169342024-02-03T17:30:06.245Z 5 100-119 157 157 157 288 288 288 F F F
169352024-02-03T17:30:06.245Z 6 120-139 154 154 154 282 282 282 F F F
169362024-02-03T17:30:06.245Z 7 140-159 151 151 151 279 279 279 F F F
169372024-02-03T17:30:06.245Z 8 160-179 149 149 149 275 275 275 F F F
169382024-02-03T17:30:06.245Z 9 180-199 154 154 154 282 282 282 F F F
169392024-02-03T17:30:06.245Z 10 200-219 156 156 156 285 285 285 F F F
169402024-02-03T17:30:06.246Z 11 220-239 147 147 147 272 272 272 F F F
169412024-02-03T17:30:06.246Z 12 240-259 154 154 154 283 283 283 F F F
169422024-02-03T17:30:06.246Z 13 260-279 148 148 148 273 273 273 F F F
169432024-02-03T17:30:06.246Z 14 280-299 155 155 155 284 284 284 F F F
169442024-02-03T17:30:06.246Z 15 300-319 151 151 151 279 279 279 F F F
169452024-02-03T17:30:06.246Z 16 320-339 156 156 156 286 286 286 F F F
169462024-02-03T17:30:06.246Z 17 340-359 152 152 152 280 280 280 F F F
169472024-02-03T17:30:06.246Z 18 360-379 156 156 156 285 285 285 F F F
169482024-02-03T17:30:06.247Z 19 380-399 157 157 157 288 288 288 F F F
169492024-02-03T17:30:06.247Z 20 400-419 156 156 156 285 285 285 F F F
169502024-02-03T17:30:06.247Z 21 420-439 156 156 156 285 285 285 F F F
169512024-02-03T17:30:06.247Z 22 440-459 154 154 154 282 282 282 F F F
169522024-02-03T17:30:06.247Z 23 460-479 156 156 156 285 285 285 F F F
169532024-02-03T17:30:06.247Z 24 480-499 156 156 156 285 285 285 F F F
169542024-02-03T17:30:06.247Z 25 500-519 156 156 156 285 285 285 F F F
169552024-02-03T17:30:06.247Z 26 520-539 149 149 149 275 275 275 F F F
169562024-02-03T17:30:06.248Z 27 540-559 156 156 156 285 285 285 F F F
169572024-02-03T17:30:06.248Z 28 560-579 156 156 156 285 285 285 F F F
169582024-02-03T17:30:06.248Z 29 580-599 156 156 156 285 285 285 F F F
169592024-02-03T17:30:06.248ZMax gen: 157, Max flush: 288
169602024-02-03T17:30:06.248Z
169612024-02-03T17:30:06.248ZBegin loop 79
169622024-02-03T17:30:06.248ZDownstairs to restart: 1
169632024-02-03T17:30:06.272Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.445799694Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767}
169642024-02-03T17:30:06.272Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.446332181Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767}
169652024-02-03T17:30:06.277Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.446399575Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767}
169662024-02-03T17:30:06.277Z{"msg":"Crucible a85b9968-dc7d-4840-a1e9-c4b18893f648 has session id: 8a8a1b20-805a-46ae-a2f5-4caff969bc16","v":0,"name":"crucible","level":30Crucible runtime is spawned
169672024-02-03T17:30:06.277Z,"time":"2024-02-03T17:30:05.446529667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169682024-02-03T17:30:06.278Z{"msg":"Upstairs opts: Upstairs UUID: a85b9968-dc7d-4840-a1e9-c4b18893f648, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.446652929Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169692024-02-03T17:30:06.278Z{"msg":"Crucible stats registered with UUID: a85b9968-dc7d-4840-a1e9-c4b18893f648","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.446719107Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169702024-02-03T17:30:06.278Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.446745563Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767}
169712024-02-03T17:30:06.278Z{"msg":"a85b9968-dc7d-4840-a1e9-c4b18893f648 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.446953503Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169722024-02-03T17:30:06.278Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447012402Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169732024-02-03T17:30:06.278Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447038953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169742024-02-03T17:30:06.278Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447097291Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169752024-02-03T17:30:06.278Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.44712464Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169762024-02-03T17:30:06.278Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447187549Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169772024-02-03T17:30:06.278Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447215936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169782024-02-03T17:30:06.278Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447243264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169792024-02-03T17:30:06.278Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447308635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169802024-02-03T17:30:06.279Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447353049Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169812024-02-03T17:30:06.279Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.44748308Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169822024-02-03T17:30:06.279Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447561329Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169832024-02-03T17:30:06.279Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.447841654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169842024-02-03T17:30:06.279Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.448901967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169852024-02-03T17:30:06.279Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.448994368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169862024-02-03T17:30:06.279Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.449039975Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169872024-02-03T17:30:06.279Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.44908007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169882024-02-03T17:30:06.279Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.449384462Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169892024-02-03T17:30:06.279Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.44945975Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169902024-02-03T17:30:06.279Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.45013039Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169912024-02-03T17:30:06.279Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.450197046Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169922024-02-03T17:30:06.279Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.450590334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169932024-02-03T17:30:06.287Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451256618Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169942024-02-03T17:30:06.288Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451354448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169952024-02-03T17:30:06.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.45144028Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169962024-02-03T17:30:06.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451522658Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169972024-02-03T17:30:06.288Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451601741Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169982024-02-03T17:30:06.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451673168Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
169992024-02-03T17:30:06.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451743338Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170002024-02-03T17:30:06.288Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451813946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170012024-02-03T17:30:06.288Z{"msg":"[0]R flush_numbers[0..12]: [281, 279, 284, 288, 287, 288, 282, 279, 275, 282, 285, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451885263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170022024-02-03T17:30:06.288Z{"msg":"[0]R generation[0..12]: [153, 151, 155, 157, 156, 157, 154, 151, 149, 154, 156, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.451961772Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs",Wait for a query_work_queue command to finish before sending IO
170032024-02-03T17:30:06.288Z"":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170042024-02-03T17:30:06.288Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452121145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170052024-02-03T17:30:06.288Z{"msg":"[1]R flush_numbers[0..12]: [281, 279, 284, 288, 287, 288, 282, 279, 275, 282, 285, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452203134Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170062024-02-03T17:30:06.288Z{"msg":"[1]R generation[0..12]: [153, 151, 155, 157, 156, 157, 154, 151, 149, 154, 156, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452331507Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170072024-02-03T17:30:06.288Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452416819Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170082024-02-03T17:30:06.288Z{"msg":"[2]R flush_numbers[0..12]: [281, 279, 284, 288, 287, 288, 282, 279, 275, 282, 285, 272]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.45249201Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170092024-02-03T17:30:06.288Z{"msg":"[2]R generation[0..12]: [153, 151, 155, 157, 156, 157, 154, 151, 149, 154, 156, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452566171Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170102024-02-03T17:30:06.289Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452641953Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170112024-02-03T17:30:06.289Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452715346Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170122024-02-03T17:30:06.289Z
170132024-02-03T17:30:06.289Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452846893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170142024-02-03T17:30:06.289Z{"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452899615Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170152024-02-03T17:30:06.289Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452924683Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170162024-02-03T17:30:06.289Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452966077Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170172024-02-03T17:30:06.289Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.452988182Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170182024-02-03T17:30:06.289Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.453029125Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170192024-02-03T17:30:06.289Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.453091279Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170202024-02-03T17:30:06.289Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.453117762Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170212024-02-03T17:30:06.289Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.453158181Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170222024-02-03T17:30:06.289Z{"msg":"a85b9968-dc7d-4840-a1e9-c4b18893f648 is now active with session: 8a8a1b20-805a-46ae-a2f5-4caff969bc16","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.453182185Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170232024-02-03T17:30:06.289Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.453220962Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767,"session_id":"8a8a1b20-805a-46ae-a2f5-4caff969bc16"}
170242024-02-03T17:30:06.289Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:05.453243503Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767}
170252024-02-03T17:30:06.289ZLoading write count information from file "/tmp/test_repair_verify.data"
170262024-02-03T17:30:06.289ZRun Repair workload
170272024-02-03T17:30:06.289Z01/30 Flush
170282024-02-03T17:30:06.289Z02/30 Write block 521 len 4096 data: 7
170292024-02-03T17:30:06.289Z03/30 Write block 260 len 36864 data: 12 13 13 12 12 13 15 12 14
170302024-02-03T17:30:06.289Z04/30 Read block 563 len 40960
170312024-02-03T17:30:06.294Z05/30 Write block 102 len 40960 data: 12 14 12 15 13 14 15 15 15 13
170322024-02-03T17:30:06.294Z06/30 Read block 460 len 8192
170332024-02-03T17:30:06.294Z07/30 Write block 591 len 8192 data: 15 13
170342024-02-03T17:30:06.294Z08/30 Read block 534 len 20480
170352024-02-03T17:30:06.294Z09/30 Read block 538 len 32768
170362024-02-03T17:30:06.295Z10/30 Read block 398 len 12288
170372024-02-03T17:30:06.299Z11/30 Write block 244 len 28672 data: 6 7 8 10 11 12 17
170382024-02-03T17:30:06.299Z12/30 Flush
170392024-02-03T17:30:06.302Z13/30 Write block 226 len 32768 data: 13 19 17 12 9 12 13 12
170402024-02-03T17:30:06.302Z14/30 Write block 380 len 4096 data: 11
170412024-02-03T17:30:06.302Z15/30 Read block 562 len 28672
170422024-02-03T17:30:06.306Z16/30 Write block 468 len 40960 data: 10 10 10 10 8 7 9 8 9 11
170432024-02-03T17:30:06.306Z17/30 Write block 10 len 24576 data: 19 16 17 14 12 11
170442024-02-03T17:30:06.311Z18/30 Write block 32 len 32768 data: 8 6 7 6 7 8 10 11
170452024-02-03T17:30:06.311Z19/30 Flush
170462024-02-03T17:30:06.316Z20/30 Flush
170472024-02-03T17:30:06.316Z21/30 Flush
170482024-02-03T17:30:06.316Z22/30 Write block 160 len 12288 data: 14 15 15
170492024-02-03T17:30:06.316Z23/30 Write block 328 len 24576 data: 11 10 12 13 12 13
170502024-02-03T17:30:06.320Z24/30 Read block 547 len 16384
170512024-02-03T17:30:06.320Z25/30 Read block 462 len 24576
170522024-02-03T17:30:06.320Z26/30 Read block 232 len 12288
170532024-02-03T17:30:06.324Z27/30 Read block 50 len 40960
170542024-02-03T17:30:06.324Z28/30 Write block 143 len 12288 data: 8 9 9
170552024-02-03T17:30:06.324Z29/30 Read block 16 len 40960
170562024-02-03T17:30:06.328Z30/30 Write block 176 len 28672 data: 12 14 12 10 11 12 11
170572024-02-03T17:30:06.328Z----------------------------------------------------------------
170582024-02-03T17:30:06.328Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
170592024-02-03T17:30:06.328ZStates: Active Active Active
170602024-02-03T17:30:06.329ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170612024-02-03T17:30:06.329Z 1 Acked 1000 Flush 0 Done Sent Done false
170622024-02-03T17:30:06.329Z 2 Acked 1001 Write 1 Done Sent Done false
170632024-02-03T17:30:06.329Z 3 Acked 1002 Write 9 Done Sent Done false
170642024-02-03T17:30:06.329Z 4 Acked 1003 Read 10 Done Sent Done false
170652024-02-03T17:30:06.329Z 5 Acked 1004 Write 10 Done Sent Done false
170662024-02-03T17:30:06.329Z 6 Acked 1005 Read 2 Done Sent Done false
170672024-02-03T17:30:06.329Z 7 Acked 1006 Write 2 Done Sent Done false
170682024-02-03T17:30:06.329Z 8 Acked 1007 Read 5 Done Sent Done false
170692024-02-03T17:30:06.329Z 9 Acked 1008 Read 8 Done Sent Done false
170702024-02-03T17:30:06.329Z 10 Acked 1009 Read 3 Done Sent Done false
170712024-02-03T17:30:06.329Z 11 Acked 1010 Write 7 Done Sent Done false
170722024-02-03T17:30:06.329Z 12 Acked 1011 Flush 0 Done Sent Done false
170732024-02-03T17:30:06.329Z 13 Acked 1012 Write 8 Done Sent Done false
170742024-02-03T17:30:06.329Z 14 Acked 1013 Write 1 Done Sent Done false
170752024-02-03T17:30:06.330Z 15 Acked 1014 Read 7 Done Sent Done false
170762024-02-03T17:30:06.330Z 16 Acked 1015 Write 10 Done Sent Done false
170772024-02-03T17:30:06.330Z 17 Acked 1016 Write 6 Done Sent Done false
170782024-02-03T17:30:06.330Z 18 Acked 1017 Write 8 Done Sent Done false
170792024-02-03T17:30:06.330Z 19 Acked 1018 Flush 0 Done Sent Done false
170802024-02-03T17:30:06.330Z 20 Acked 1019 Flush 0 Done Sent Done false
170812024-02-03T17:30:06.330Z 21 Acked 1020 Flush 0 Done Sent Done false
170822024-02-03T17:30:06.330Z 22 Acked 1021 Write 3 Done Sent Done false
170832024-02-03T17:30:06.330Z 23 Acked 1022 Write 6 Done Sent Done false
170842024-02-03T17:30:06.330Z 24 Acked 1023 Read 4 Done Sent Done false
170852024-02-03T17:30:06.330Z 25 Acked 1024 Read 6 Done Sent Done false
170862024-02-03T17:30:06.330Z 26 Acked 1025 Read 3 Done Sent Done false
170872024-02-03T17:30:06.330Z 27 Acked 1026 Read 10 Done Sent Done false
170882024-02-03T17:30:06.330Z 28 Acked 1027 Write 3 Done Sent Done false
170892024-02-03T17:30:06.331Z 29 Acked 1028 Read 10 Done Sent Done false
170902024-02-03T17:30:06.331Z 30 Acked 1029 Write 7 Sent Sent Sent false
170912024-02-03T17:30:06.331Z STATES DS:0 DS:1 DS:2 TOTAL
170922024-02-03T17:30:06.331Z New 0 0 0 0
170932024-02-03T17:30:06.331Z Sent 1 30 1 32
170942024-02-03T17:30:06.331Z Done 29 0 29 58
170952024-02-03T17:30:06.331Z Skipped 0 0 0 0
170962024-02-03T17:30:06.331Z Error 0 0 0 0
170972024-02-03T17:30:06.331ZLast Flush: 1020 0 1020
170982024-02-03T17:30:06.331ZDownstairs last five completed:
170992024-02-03T17:30:06.331ZUpstairs last five completed: 30 29 28 27 26
171002024-02-03T17:30:06.331Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:05.502119948Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1767}
171012024-02-03T17:30:06.337ZWrote out file "/tmp/test_repair_verify.data"
171022024-02-03T17:30:06.337Z
171032024-02-03T17:30:06.345ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171042024-02-03T17:30:06.371ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171052024-02-03T17:30:06.371Z 0 000-019 158 153 158 291 281 291 F F F <---
171062024-02-03T17:30:06.371Z 1 020-039 158 151 158 291 279 291 F F F <---
171072024-02-03T17:30:06.371Z 2 040-059 155 155 155 284 284 284 F F F
171082024-02-03T17:30:06.372Z 3 060-079 157 157 157 288 288 288 F F F
171092024-02-03T17:30:06.372Z 4 080-099 156 156 156 287 287 287 F F F
171102024-02-03T17:30:06.377Z 5 100-119 158 157 158 290 288 290 F F F <---
171112024-02-03T17:30:06.377Z 6 120-139 154 154 154 282 282 282 F F F
171122024-02-03T17:30:06.377Z 7 140-159 151 151 151 279 279 279 T F T <---
171132024-02-03T17:30:06.377Z 8 160-179 149 149 149 275 275 275 T F T <---
171142024-02-03T17:30:06.377Z 9 180-199 154 154 154 282 282 282 T F T <---
171152024-02-03T17:30:06.377Z 10 200-219 156 156 156 285 285 285 F F F
171162024-02-03T17:30:06.377Z 11 220-239 158 147 158 291 272 291 F F F <---
171172024-02-03T17:30:06.377Z 12 240-259 158 154 158 290 283 290 F F F <---
171182024-02-03T17:30:06.377Z 13 260-279 158 148 158 290 273 290 F F F <---
171192024-02-03T17:30:06.378Z 14 280-299 155 155 155 284 284 284 F F F
171202024-02-03T17:30:06.378Z 15 300-319 151 151 151 279 279 279 F F F
171212024-02-03T17:30:06.378Z 16 320-339 156 156 156 286 286 286 T F T <---
171222024-02-03T17:30:06.378Z 17 340-359 152 152 152 280 280 280 F F F
171232024-02-03T17:30:06.378Z 18 360-379 156 156 156 285 285 285 F F F
171242024-02-03T17:30:06.378Z 19 380-399 158 157 158 291 288 291 F F F <---
171252024-02-03T17:30:06.378Z 20 400-419 156 156 156 285 285 285 F F F
171262024-02-03T17:30:06.378Z 21 420-439 156 156 156 285 285 285 F F F
171272024-02-03T17:30:06.378Z 22 440-459 154 154 154 282 282 282 F F F
171282024-02-03T17:30:06.378Z 23 460-479 158 156 158 291 285 291 F F F <---
171292024-02-03T17:30:06.378Z 24 480-499 156 156 156 285 285 285 F F F
171302024-02-03T17:30:06.378Z 25 500-519 156 156 156 285 285 285 F F F
171312024-02-03T17:30:06.378Z 26 520-539 158 149 158 290 275 290 F F F <---
171322024-02-03T17:30:06.378Z 27 540-559 156 156 156 285 285 285 F F F
171332024-02-03T17:30:06.378Z 28 560-579 156 156 156 285 285 285 F F F
171342024-02-03T17:30:06.379Z 29 580-599 158 156 158 290 285 290 F F F <---
171352024-02-03T17:30:06.379ZMax gen: 158, Max flush: 291
171362024-02-03T17:30:06.379ZError: Difference in extent metadata found!
171372024-02-03T17:30:06.379ZA Difference in extent metadata is expected here
171382024-02-03T17:30:06.379ZOn loop 79
171392024-02-03T17:30:06.379Z
171402024-02-03T17:30:06.384ZVerifying data now
171412024-02-03T17:30:11.606ZNo change after verify
171422024-02-03T17:30:11.609ZLoop: 79 Downstairs dump after verify (and repair):
171432024-02-03T17:30:11.631ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171442024-02-03T17:30:11.631Z 0 000-019 158 158 158 291 291 291 F F F
171452024-02-03T17:30:11.634Z 1 020-039 158 158 158 291 291 291 F F F
171462024-02-03T17:30:11.634Z 2 040-059 155 155 155 284 284 284 F F F
171472024-02-03T17:30:11.634Z 3 060-079 157 157 157 288 288 288 F F F
171482024-02-03T17:30:11.634Z 4 080-099 156 156 156 287 287 287 F F F
171492024-02-03T17:30:11.634Z 5 100-119 158 158 158 290 290 290 F F F
171502024-02-03T17:30:11.635Z 6 120-139 154 154 154 282 282 282 F F F
171512024-02-03T17:30:11.635Z 7 140-159 159 159 159 292 292 292 F F F
171522024-02-03T17:30:11.635Z 8 160-179 159 159 159 292 292 292 F F F
171532024-02-03T17:30:11.635Z 9 180-199 159 159 159 292 292 292 F F F
171542024-02-03T17:30:11.635Z 10 200-219 156 156 156 285 285 285 F F F
171552024-02-03T17:30:11.635Z 11 220-239 158 158 158 291 291 291 F F F
171562024-02-03T17:30:11.635Z 12 240-259 158 158 158 290 290 290 F F F
171572024-02-03T17:30:11.635Z 13 260-279 158 158 158 290 290 290 F F F
171582024-02-03T17:30:11.635Z 14 280-299 155 155 155 284 284 284 F F F
171592024-02-03T17:30:11.635Z 15 300-319 151 151 151 279 279 279 F F F
171602024-02-03T17:30:11.635Z 16 320-339 159 159 159 292 292 292 F F F
171612024-02-03T17:30:11.635Z 17 340-359 152 152 152 280 280 280 F F F
171622024-02-03T17:30:11.635Z 18 360-379 156 156 156 285 285 285 F F F
171632024-02-03T17:30:11.635Z 19 380-399 158 158 158 291 291 291 F F F
171642024-02-03T17:30:11.635Z 20 400-419 156 156 156 285 285 285 F F F
171652024-02-03T17:30:11.635Z 21 420-439 156 156 156 285 285 285 F F F
171662024-02-03T17:30:11.635Z 22 440-459 154 154 154 282 282 282 F F F
171672024-02-03T17:30:11.635Z 23 460-479 158 158 158 291 291 291 F F F
171682024-02-03T17:30:11.635Z 24 480-499 156 156 156 285 285 285 F F F
171692024-02-03T17:30:11.635Z 25 500-519 156 156 156 285 285 285 F F F
171702024-02-03T17:30:11.635Z 26 520-539 158 158 158 290 290 290 F F F
171712024-02-03T17:30:11.635Z 27 540-559 156 156 156 285 285 285 F F F
171722024-02-03T17:30:11.635Z 28 560-579 156 156 156 285 285 285 F F F
171732024-02-03T17:30:11.635Z 29 580-599 158 158 158 290 290 290 F F F
171742024-02-03T17:30:11.636ZMax gen: 159, Max flush: 292
171752024-02-03T17:30:11.639Z
171762024-02-03T17:30:11.639ZBegin loop 80
171772024-02-03T17:30:11.639ZDownstairs to restart: 0
171782024-02-03T17:30:11.671Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.845445345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775}
171792024-02-03T17:30:11.671Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846030654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775}
171802024-02-03T17:30:11.671Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846059585Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775}
171812024-02-03T17:30:11.675Z{"msg":"Crucible 567291ed-f82c-4794-9387-252fb55448da has session id: 16da6cda-62ac-4331-bc5f-ee79af211fb2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846124344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171822024-02-03T17:30:11.675Z{"msg":"Upstairs opts: Upstairs UUID: 567291ed-f82c-4794-9387-252fb55448da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846196444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171832024-02-03T17:30:11.675Z{"msg":"Crucible stats registered with UUID: 567291ed-f82c-4794-9387-252fb55448da","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846235731Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171842024-02-03T17:30:11.675ZCrucible runtime is spawned
171852024-02-03T17:30:11.675Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846508737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775}
171862024-02-03T17:30:11.675Z{"msg":"567291ed-f82c-4794-9387-252fb55448da active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846564152Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171872024-02-03T17:30:11.675Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846588703Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171882024-02-03T17:30:11.675Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846612419Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171892024-02-03T17:30:11.675Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846653982Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171902024-02-03T17:30:11.675Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846678672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171912024-02-03T17:30:11.676Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.84670099Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171922024-02-03T17:30:11.676Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846770999Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171932024-02-03T17:30:11.676Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846914982Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171942024-02-03T17:30:11.676Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.846984942Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171952024-02-03T17:30:11.676Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.847065264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171962024-02-03T17:30:11.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.847315129Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171972024-02-03T17:30:11.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.84737265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171982024-02-03T17:30:11.676Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.847416758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
171992024-02-03T17:30:11.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.848808984Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172002024-02-03T17:30:11.676Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.848990765Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172012024-02-03T17:30:11.676Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.849102413Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172022024-02-03T17:30:11.676Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.849131513Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172032024-02-03T17:30:11.676Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.849812893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172042024-02-03T17:30:11.681Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.850003856Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172052024-02-03T17:30:11.681Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.850119135Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172062024-02-03T17:30:11.681Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.850265646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172072024-02-03T17:30:11.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.850893203Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172082024-02-03T17:30:11.681Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.850991172Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172092024-02-03T17:30:11.681Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.851036872Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172102024-02-03T17:30:11.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.851354752Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172112024-02-03T17:30:11.681Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.851425631Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172122024-02-03T17:30:11.681Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.851451738Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172132024-02-03T17:30:11.681Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.851752259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172142024-02-03T17:30:11.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852465378Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172152024-02-03T17:30:11.681Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852527654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172162024-02-03T17:30:11.681Z{"msg":"[0]R flush_numbers[0..12]: [291, 291, 284, 288, 287, 290, 282, 292, 292, 292, 285, 291]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852555543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172172024-02-03T17:30:11.681Z{"msg":"[0]R generation[0..12]: [158, 158, 155, 157, 156, 158, 154, 159, 159, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852581106Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172182024-02-03T17:30:11.681Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852624782Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172192024-02-03T17:30:11.681Z{"msg":"[1]R flush_numbers[0..12]: [291, 291, 284, 288, 287, 290, 282, 292, 292, 292, 285, 291]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852649937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172202024-02-03T17:30:11.681Z{"msg":"[1]R generation[0..12]: [158, 158, 155, 157, 156, 158, 154, 159, 159, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852692194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172212024-02-03T17:30:11.681Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852717707Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172222024-02-03T17:30:11.681Z{"msg":"[2]R flush_numbers[0..12]: [291, 291, 284, 288, 287, 290, 282, 292, 292, 292, 285, 291]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852759605Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172232024-02-03T17:30:11.681Z{"msg":"[2]R generation[0..12]: [158, 158, 155, 157, 156, 158, 154, 159, 159, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852784393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172242024-02-03T17:30:11.681Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852826992Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
172252024-02-03T17:30:11.681Z16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172262024-02-03T17:30:11.682Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852854071Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172272024-02-03T17:30:11.682Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852895318Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172282024-02-03T17:30:11.682Z{"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852918788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172292024-02-03T17:30:11.682Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852966497Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172302024-02-03T17:30:11.682Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.852990316Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172312024-02-03T17:30:11.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.853036491Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172322024-02-03T17:30:11.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.853060302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172332024-02-03T17:30:11.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.853083459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172342024-02-03T17:30:11.682Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.853123933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172352024-02-03T17:30:11.682Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.853146261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172362024-02-03T17:30:11.682Z{"msg":"567291ed-f82c-4794-9387-252fb55448da is now active with session: 16da6cda-62ac-4331-bc5f-ee79af211fb2","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.853192817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172372024-02-03T17:30:11.682Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.853214453Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775,"session_id":"16da6cda-62ac-4331-bc5f-ee79af211fb2"}
172382024-02-03T17:30:11.682Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:10.85323853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775}
172392024-02-03T17:30:11.682ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172402024-02-03T17:30:11.682ZLoading write count information from file "/tmp/test_repair_verify.data"
172412024-02-03T17:30:11.682ZRun Repair workload
172422024-02-03T17:30:11.682Z01/30 Write block 27 len 8192 data: 10 12
172432024-02-03T17:30:11.686Z02/30 Read block 36 len 20480
172442024-02-03T17:30:11.686Z03/30 Read block 36 len 8192
172452024-02-03T17:30:11.686Z04/30 Write block 581 len 36864 data: 18 18 16 17 15 18 14 14 16
172462024-02-03T17:30:11.692Z05/30 Flush
172472024-02-03T17:30:11.692Z06/30 Write block 263 len 8192 data: 13 13
172482024-02-03T17:30:11.695Z07/30 Write block 340 len 16384 data: 6 5 9 10
172492024-02-03T17:30:11.695Z08/30 Flush
172502024-02-03T17:30:11.695Z09/30 Write block 259 len 20480 data: 9 13 14 14 14
172512024-02-03T17:30:11.695Z10/30 Flush
172522024-02-03T17:30:11.698Z11/30 Write block 174 len 24576 data: 10 11 13 15 13 11
172532024-02-03T17:30:11.702Z12/30 Write block 365 len 16384 data: 13 10 11 10
172542024-02-03T17:30:11.702Z13/30 Write block 2 len 32768 data: 9 10 12 14 14 15 15 15
172552024-02-03T17:30:11.702Z14/30 Read block 115 len 24576
172562024-02-03T17:30:11.705Z15/30 Read block 491 len 8192
172572024-02-03T17:30:11.705Z16/30 Read block 294 len 28672
172582024-02-03T17:30:11.709Z17/30 Read block 252 len 40960
172592024-02-03T17:30:11.709Z18/30 Write block 547 len 16384 data: 12 17 18 15
172602024-02-03T17:30:11.709Z19/30 Read block 200 len 8192
172612024-02-03T17:30:11.712Z20/30 Write block 422 len 4096 data: 14
172622024-02-03T17:30:11.712Z21/30 Read block 546 len 4096
172632024-02-03T17:30:11.712Z22/30 Write block 103 len 20480 data: 15 13 16 14 15
172642024-02-03T17:30:11.715Z23/30 Write block 369 len 12288 data: 9 8 10
172652024-02-03T17:30:11.715Z24/30 Read block 127 len 4096
172662024-02-03T17:30:11.715Z25/30 Read block 580 len 16384
172672024-02-03T17:30:11.719Z26/30 Read block 103 len 36864
172682024-02-03T17:30:11.719Z27/30 Read block 28 len 28672
172692024-02-03T17:30:11.722Z28/30 Write block 8 len 8192 data: 16 16
172702024-02-03T17:30:11.722Z29/30 Write block 511 len 40960 data: 13 13 11 8 10 8 11 10 7 7
172712024-02-03T17:30:11.722Z30/30 Read block 550 len 20480
172722024-02-03T17:30:11.726Z----------------------------------------------------------------
172732024-02-03T17:30:11.726Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172742024-02-03T17:30:11.726ZStates: Active Active Active
172752024-02-03T17:30:11.726ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172762024-02-03T17:30:11.726Z 1 Acked 1000 Write 2 Done Done Done false
172772024-02-03T17:30:11.726Z 2 Acked 1001 Read 5 Done Done Done false
172782024-02-03T17:30:11.726Z 3 Acked 1002 Read 2 Sent Done Done false
172792024-02-03T17:30:11.726Z 4 Acked 1003 Write 9 Sent Done Done false
172802024-02-03T17:30:11.726Z 5 Acked 1004 Flush 0 Sent Done Done false
172812024-02-03T17:30:11.726Z 6 Acked 1005 Write 2 Sent Done Done false
172822024-02-03T17:30:11.726Z 7 Acked 1006 Write 4 Sent Done Done false
172832024-02-03T17:30:11.726Z 8 Acked 1007 Flush 0 Sent Done Done false
172842024-02-03T17:30:11.726Z 9 Acked 1008 Write 5 Sent Done Done false
172852024-02-03T17:30:11.726Z 10 Acked 1009 Flush 0 Sent Done Done false
172862024-02-03T17:30:11.726Z 11 Acked 1010 Write 6 Sent Done Done false
172872024-02-03T17:30:11.726Z 12 Acked 1011 Write 4 Sent Done Done false
172882024-02-03T17:30:11.726Z 13 Acked 1012 Write 8 Sent Done Done false
172892024-02-03T17:30:11.726Z 14 Acked 1013 Read 6 Sent Done Done false
172902024-02-03T17:30:11.726Z 15 Acked 1014 Read 2 Sent Done Done false
172912024-02-03T17:30:11.726Z 16 Acked 1015 Read 7 Sent Done Done false
172922024-02-03T17:30:11.726Z 17 Acked 1016 Read 10 Sent Done Done false
172932024-02-03T17:30:11.726Z 18 Acked 1017 Write 4 Sent Done Done false
172942024-02-03T17:30:11.727Z 19 Acked 1018 Read 2 Sent Done Done false
172952024-02-03T17:30:11.727Z 20 Acked 1019 Write 1 Sent Done Done false
172962024-02-03T17:30:11.727Z 21 Acked 1020 Read 1 Sent Done Done false
172972024-02-03T17:30:11.727Z 22 Acked 1021 Write 5 Sent Done Done false
172982024-02-03T17:30:11.727Z 23 Acked 1022 Write 3 Sent Done Done false
172992024-02-03T17:30:11.727Z 24 Acked 1023 Read 1 Sent Done Done false
173002024-02-03T17:30:11.727Z 25 Acked 1024 Read 4 Sent Done Done false
173012024-02-03T17:30:11.727Z 26 Acked 1025 Read 9 Sent Done Done false
173022024-02-03T17:30:11.727Z 27 Acked 1026 Read 7 Sent Done Done false
173032024-02-03T17:30:11.727Z 28 Acked 1027 Write 2 Sent Done Done false
173042024-02-03T17:30:11.727Z 29 Acked 1028 Write 10 Sent Done Done false
173052024-02-03T17:30:11.727Z 30 Acked 1029 Read 5 Sent Sent Done false
173062024-02-03T17:30:11.727Z STATES DS:0 DS:1 DS:2 TOTAL
173072024-02-03T17:30:11.727Z New 0 0 0 0
173082024-02-03T17:30:11.727Z Sent 28 1 0 29
173092024-02-03T17:30:11.727Z Done 2 29 30 61
173102024-02-03T17:30:11.727Z Skipped 0 0 0 0
173112024-02-03T17:30:11.727Z Error 0 0 0 0
173122024-02-03T17:30:11.727ZLast Flush: 0 1009 1009
173132024-02-03T17:30:11.727ZDownstairs last five completed:
173142024-02-03T17:30:11.727ZUpstairs last five completed: 30 29 28 27 26
173152024-02-03T17:30:11.727Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:10.898868748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1775}
173162024-02-03T17:30:11.727ZWrote out file "/tmp/test_repair_verify.data"
173172024-02-03T17:30:11.732Z
173182024-02-03T17:30:11.740ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173192024-02-03T17:30:11.769ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173202024-02-03T17:30:11.769Z 0 000-019 158 158 158 291 291 291 F T T <---
173212024-02-03T17:30:11.773Z 1 020-039 158 160 160 291 293 293 T F F <---
173222024-02-03T17:30:11.773Z 2 040-059 155 155 155 284 284 284 F F F
173232024-02-03T17:30:11.773Z 3 060-079 157 157 157 288 288 288 F F F
173242024-02-03T17:30:11.773Z 4 080-099 156 156 156 287 287 287 F F F
173252024-02-03T17:30:11.773Z 5 100-119 158 158 158 290 290 290 F T T <---
173262024-02-03T17:30:11.773Z 6 120-139 154 154 154 282 282 282 F F F
173272024-02-03T17:30:11.773Z 7 140-159 159 159 159 292 292 292 F F F
173282024-02-03T17:30:11.773Z 8 160-179 159 159 159 292 292 292 F T T <---
173292024-02-03T17:30:11.773Z 9 180-199 159 159 159 292 292 292 F F F
173302024-02-03T17:30:11.773Z 10 200-219 156 156 156 285 285 285 F F F
173312024-02-03T17:30:11.773Z 11 220-239 158 158 158 291 291 291 F F F
173322024-02-03T17:30:11.773Z 12 240-259 158 160 160 290 295 295 F F F <---
173332024-02-03T17:30:11.773Z 13 260-279 158 160 160 290 295 295 F F F <---
173342024-02-03T17:30:11.773Z 14 280-299 155 155 155 284 284 284 F F F
173352024-02-03T17:30:11.773Z 15 300-319 151 151 151 279 279 279 F F F
173362024-02-03T17:30:11.773Z 16 320-339 159 159 159 292 292 292 F F F
173372024-02-03T17:30:11.773Z 17 340-359 152 160 160 280 294 294 F F F <---
173382024-02-03T17:30:11.773Z 18 360-379 156 156 156 285 285 285 F T T <---
173392024-02-03T17:30:11.773Z 19 380-399 158 158 158 291 291 291 F F F
173402024-02-03T17:30:11.774Z 20 400-419 156 156 156 285 285 285 F F F
173412024-02-03T17:30:11.774Z 21 420-439 156 156 156 285 285 285 F T T <---
173422024-02-03T17:30:11.774Z 22 440-459 154 154 154 282 282 282 F F F
173432024-02-03T17:30:11.774Z 23 460-479 158 158 158 291 291 291 F F F
173442024-02-03T17:30:11.774Z 24 480-499 156 156 156 285 285 285 F F F
173452024-02-03T17:30:11.774Z 25 500-519 156 156 156 285 285 285 F T T <---
173462024-02-03T17:30:11.774Z 26 520-539 158 158 158 290 290 290 F T T <---
173472024-02-03T17:30:11.774Z 27 540-559 156 156 156 285 285 285 F T T <---
173482024-02-03T17:30:11.774Z 28 560-579 156 156 156 285 285 285 F F F
173492024-02-03T17:30:11.774Z 29 580-599 158 160 160 290 293 293 F F F <---
173502024-02-03T17:30:11.774ZMax gen: 160, Max flush: 295
173512024-02-03T17:30:11.774ZError: Difference in extent metadata found!
173522024-02-03T17:30:11.778ZA Difference in extent metadata is expected here
173532024-02-03T17:30:11.778ZOn loop 80
173542024-02-03T17:30:11.778Z
173552024-02-03T17:30:11.778ZVerifying data now
173562024-02-03T17:30:16.939ZNo change after verify
173572024-02-03T17:30:16.940ZLoop: 80 Downstairs dump after verify (and repair):
173582024-02-03T17:30:16.963ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173592024-02-03T17:30:16.963Z 0 000-019 161 161 161 296 296 296 F F F
173602024-02-03T17:30:16.963Z 1 020-039 160 160 160 293 293 293 F F F
173612024-02-03T17:30:16.963Z 2 040-059 155 155 155 284 284 284 F F F
173622024-02-03T17:30:16.963Z 3 060-079 157 157 157 288 288 288 F F F
173632024-02-03T17:30:16.963Z 4 080-099 156 156 156 287 287 287 F F F
173642024-02-03T17:30:16.963Z 5 100-119 161 161 161 296 296 296 F F F
173652024-02-03T17:30:16.963Z 6 120-139 154 154 154 282 282 282 F F F
173662024-02-03T17:30:16.963Z 7 140-159 159 159 159 292 292 292 F F F
173672024-02-03T17:30:16.963Z 8 160-179 161 161 161 296 296 296 F F F
173682024-02-03T17:30:16.964Z 9 180-199 159 159 159 292 292 292 F F F
173692024-02-03T17:30:16.967Z 10 200-219 156 156 156 285 285 285 F F F
173702024-02-03T17:30:16.967Z 11 220-239 158 158 158 291 291 291 F F F
173712024-02-03T17:30:16.967Z 12 240-259 160 160 160 295 295 295 F F F
173722024-02-03T17:30:16.967Z 13 260-279 160 160 160 295 295 295 F F F
173732024-02-03T17:30:16.967Z 14 280-299 155 155 155 284 284 284 F F F
173742024-02-03T17:30:16.968Z 15 300-319 151 151 151 279 279 279 F F F
173752024-02-03T17:30:16.968Z 16 320-339 159 159 159 292 292 292 F F F
173762024-02-03T17:30:16.968Z 17 340-359 160 160 160 294 294 294 F F F
173772024-02-03T17:30:16.968Z 18 360-379 161 161 161 296 296 296 F F F
173782024-02-03T17:30:16.968Z 19 380-399 158 158 158 291 291 291 F F F
173792024-02-03T17:30:16.968Z 20 400-419 156 156 156 285 285 285 F F F
173802024-02-03T17:30:16.968Z 21 420-439 161 161 161 296 296 296 F F F
173812024-02-03T17:30:16.968Z 22 440-459 154 154 154 282 282 282 F F F
173822024-02-03T17:30:16.968Z 23 460-479 158 158 158 291 291 291 F F F
173832024-02-03T17:30:16.968Z 24 480-499 156 156 156 285 285 285 F F F
173842024-02-03T17:30:16.968Z 25 500-519 161 161 161 296 296 296 F F F
173852024-02-03T17:30:16.968Z 26 520-539 161 161 161 296 296 296 F F F
173862024-02-03T17:30:16.968Z 27 540-559 161 161 161 296 296 296 F F F
173872024-02-03T17:30:16.968Z 28 560-579 156 156 156 285 285 285 F F F
173882024-02-03T17:30:16.968Z 29 580-599 160 160 160 293 293 293 F F F
173892024-02-03T17:30:16.968ZMax gen: 161, Max flush: 296
173902024-02-03T17:30:16.972Z
173912024-02-03T17:30:16.972ZBegin loop 81
173922024-02-03T17:30:16.972ZDownstairs to restart: 0
173932024-02-03T17:30:17.003Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.17586556Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783}
173942024-02-03T17:30:17.003Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.176424529Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783}
173952024-02-03T17:30:17.009Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.17649588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783}
173962024-02-03T17:30:17.009Z{"msg":"Crucible ca4ef2dd-3db5-4ff9-98bd-656fe58c1401 has session id: e20ef45f-2cc5-4218-9e12-aa27ce90d4db","v":0,"name":"crucible","level":30Crucible runtime is spawned
173972024-02-03T17:30:17.009Z,"time":"2024-02-03T17:30:16.17656714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
173982024-02-03T17:30:17.009Z{"msg":"Upstairs opts: Upstairs UUID: ca4ef2dd-3db5-4ff9-98bd-656fe58c1401, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.176713375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
173992024-02-03T17:30:17.009Z{"msg":"Crucible stats registered with UUID: ca4ef2dd-3db5-4ff9-98bd-656fe58c1401","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.176823784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174002024-02-03T17:30:17.009Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.176851487Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783}
174012024-02-03T17:30:17.009Z{"msg":"ca4ef2dd-3db5-4ff9-98bd-656fe58c1401 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.176927945Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174022024-02-03T17:30:17.009Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.176988865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174032024-02-03T17:30:17.009Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177080758Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174042024-02-03T17:30:17.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177178295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174052024-02-03T17:30:17.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177226649Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174062024-02-03T17:30:17.010Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177297167Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174072024-02-03T17:30:17.010Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177391007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174082024-02-03T17:30:17.010Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.17743485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174092024-02-03T17:30:17.010Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177558499Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174102024-02-03T17:30:17.010Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177610282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174112024-02-03T17:30:17.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.17767516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174122024-02-03T17:30:17.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177702084Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174132024-02-03T17:30:17.010Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.177731971Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174142024-02-03T17:30:17.010Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.179250672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174152024-02-03T17:30:17.010Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.179319644Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174162024-02-03T17:30:17.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.179345755Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174172024-02-03T17:30:17.011Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.179367791Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174182024-02-03T17:30:17.011Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.180162158Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174192024-02-03T17:30:17.011Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.1802282Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174202024-02-03T17:30:17.011Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.180536436Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174212024-02-03T17:30:17.011Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.180729471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174222024-02-03T17:30:17.011Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.181146294Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174232024-02-03T17:30:17.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.181404331Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174242024-02-03T17:30:17.011Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.181462431Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174252024-02-03T17:30:17.011Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.181487989Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174262024-02-03T17:30:17.011Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.181618913Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174272024-02-03T17:30:17.011Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.18167969Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174282024-02-03T17:30:17.011Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.181706117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174292024-02-03T17:30:17.012Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182037668Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174302024-02-03T17:30:17.012Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182100269Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174312024-02-03T17:30:17.012Z{"msg":"[0]R flush_numbers[0..12]: [296, 293, 284, 288, 287, 296, 282, 292, 296, 292, 285, 291]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182146176Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174322024-02-03T17:30:17.012Z{"msg":"[0]R generation[0..12]: [161, 160, 155, 157, 156, 161, 154, 159, 161, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182210436Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174332024-02-03T17:30:17.012Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182257776Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174342024-02-03T17:30:17.012Z{"msg":"[1]R flush_numbers[0..12]: [296, 293, 284, 288, 287, 296, 282, 292, 296, 292, 285, 291]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182317546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174352024-02-03T17:30:17.016Z{"msg":"[1]R generation[0..12]: [161, 160, 155, 157, 156, 161, 154, 159, 161, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182361121Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174362024-02-03T17:30:17.017ZWait for a query_work_queue command to finish before sending IO
174372024-02-03T17:30:17.017Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182422643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174382024-02-03T17:30:17.017Z{"msg":"[2]R flush_numbers[0..12]: [296, 293, 284, 288, 287, 296, 282, 292, 296, 292, 285, 291]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.18246764Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174392024-02-03T17:30:17.017Z{"msg":"[2]R generation[0..12]: [161, 160, 155, 157, 156, 161, 154, 159, 161, 159, 156, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182598516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174402024-02-03T17:30:17.017Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182645918Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174412024-02-03T17:30:17.017Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.18270072Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174422024-02-03T17:30:17.017Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182744186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174432024-02-03T17:30:17.017Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182798224Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174442024-02-03T17:30:17.017Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182840565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174452024-02-03T17:30:17.017Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182894783Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174462024-02-03T17:30:17.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182933556Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174472024-02-03T17:30:17.017Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.182992579Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174482024-02-03T17:30:17.018Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.183034248Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174492024-02-03T17:30:17.018Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.183085586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174502024-02-03T17:30:17.018Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.183109629Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174512024-02-03T17:30:17.018Z{"msg":"ca4ef2dd-3db5-4ff9-98bd-656fe58c1401 is now active with session: e20ef45f-2cc5-4218-9e12-aa27ce90d4db","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.183131094Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174522024-02-03T17:30:17.018Z{"msg":"Set Active after no reconciliation"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174532024-02-03T17:30:17.018Z,"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.18317492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783,"session_id":"e20ef45f-2cc5-4218-9e12-aa27ce90d4db"}
174542024-02-03T17:30:17.018Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:16.183198051Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783}
174552024-02-03T17:30:17.018ZLoading write count information from file "/tmp/test_repair_verify.data"
174562024-02-03T17:30:17.018ZRun Repair workload
174572024-02-03T17:30:17.018Z01/30 Write block 368 len 36864 data: 11 10 9 11 9 8 9 9 12
174582024-02-03T17:30:17.018Z02/30 Write block 400 len 40960 data: 12 10 10 13 10 15 13 14 11 12
174592024-02-03T17:30:17.018Z03/30 Read block 262 len 16384
174602024-02-03T17:30:17.022Z04/30 Read block 494 len 28672
174612024-02-03T17:30:17.022Z05/30 Read block 214 len 16384
174622024-02-03T17:30:17.022Z06/30 Flush
174632024-02-03T17:30:17.026Z07/30 Write block 273 len 24576 data: 15 18 20 20 19 19
174642024-02-03T17:30:17.026Z08/30 Write block 496 len 36864 data: 13 12 15 17 17 19 16 17 17
174652024-02-03T17:30:17.026Z09/30 Write block 561 len 4096 data: 8
174662024-02-03T17:30:17.026Z10/30 Write block 562 len 4096 data: 5
174672024-02-03T17:30:17.031Z11/30 Write block 201 len 24576 data: 8 9 11 11 9 8
174682024-02-03T17:30:17.031Z12/30 Read block 72 len 40960
174692024-02-03T17:30:17.035Z13/30 Write block 570 len 20480 data: 11 10 6 7 7
174702024-02-03T17:30:17.035Z14/30 Write block 99 len 40960 data: 11 13 13 13 16 14 17 15 16 16
174712024-02-03T17:30:17.035Z15/30 Write block 180 len 36864 data: 12 13 12 10 8 9 9 11 10
174722024-02-03T17:30:17.035Z16/30 Write block 164 len 32768 data: 11 9 8 8 7 11 11 11
174732024-02-03T17:30:17.042Z17/30 Read block 130 len 12288
174742024-02-03T17:30:17.042Z18/30 Write block 525 len 12288 data: 9 12 14
174752024-02-03T17:30:17.043Z19/30 Read block 306 len 28672
174762024-02-03T17:30:17.049Z20/30 Write block 214 len 40960 data: 10 12 10 10 9 11 12 17 16 14
174772024-02-03T17:30:17.049Z21/30 Read block 231 len 40960
174782024-02-03T17:30:17.049Z22/30 Write block 231 len 40960 data: 13 14 13 13 14 13 15 11 13 10
174792024-02-03T17:30:17.049Z23/30 Read block 187 len 20480
174802024-02-03T17:30:17.053Z24/30 Read block 159 len 8192
174812024-02-03T17:30:17.053Z25/30 Read block 447 len 16384
174822024-02-03T17:30:17.053Z26/30 Read block 539 len 8192
174832024-02-03T17:30:17.053Z27/30 Read block 332 len 12288
174842024-02-03T17:30:17.057Z28/30 Write block 189 len 20480 data: 11 10 10 8 6
174852024-02-03T17:30:17.058Z29/30 Write block 483 len 8192 data: 10 8
174862024-02-03T17:30:17.058Z30/30 Read block 28 len 4096
174872024-02-03T17:30:17.058Z----------------------------------------------------------------
174882024-02-03T17:30:17.058Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
174892024-02-03T17:30:17.058ZStates: Active Active Active
174902024-02-03T17:30:17.058ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174912024-02-03T17:30:17.059Z 1 Acked 1000 Write 9 Done Done Done false
174922024-02-03T17:30:17.059Z 2 Acked 1001 Write 10 Done Done Done false
174932024-02-03T17:30:17.059Z 3 Acked 1002 Read 4 Sent Done Done false
174942024-02-03T17:30:17.059Z 4 Acked 1003 Read 7 Sent Done Done false
174952024-02-03T17:30:17.059Z 5 Acked 1004 Read 4 Sent Done Done false
174962024-02-03T17:30:17.059Z 6 Acked 1005 Flush 0 Sent Done Done false
174972024-02-03T17:30:17.059Z 7 Acked 1006 Write 6 Sent Done Done false
174982024-02-03T17:30:17.059Z 8 Acked 1007 Write 9 Sent Done Done false
174992024-02-03T17:30:17.059Z 9 Acked 1008 Write 1 Sent Done Done false
175002024-02-03T17:30:17.059Z 10 Acked 1009 Write 1 Sent Done Done false
175012024-02-03T17:30:17.059Z 11 Acked 1010 Write 6 Sent Done Done false
175022024-02-03T17:30:17.059Z 12 Acked 1011 Read 10 Sent Done Done false
175032024-02-03T17:30:17.059Z 13 Acked 1012 Write 5 Sent Done Done false
175042024-02-03T17:30:17.060Z 14 Acked 1013 Write 10 Sent Done Done false
175052024-02-03T17:30:17.060Z 15 Acked 1014 Write 9 Sent Done Done false
175062024-02-03T17:30:17.060Z 16 Acked 1015 Write 8 Sent Done Done false
175072024-02-03T17:30:17.060Z 17 Acked 1016 Read 3 Sent Done Done false
175082024-02-03T17:30:17.060Z 18 Acked 1017 Write 3 Sent Done Done false
175092024-02-03T17:30:17.060Z 19 Acked 1018 Read 7 Sent Done Done false
175102024-02-03T17:30:17.060Z 20 Acked 1019 Write 10 Sent Done Done false
175112024-02-03T17:30:17.060Z 21 Acked 1020 Read 10 Sent Done Done false
175122024-02-03T17:30:17.060Z 22 Acked 1021 Write 10 Sent Done Done false
175132024-02-03T17:30:17.060Z 23 Acked 1022 Read 5 Sent Done Done false
175142024-02-03T17:30:17.060Z 24 Acked 1023 Read 2 Sent Done Done false
175152024-02-03T17:30:17.060Z 25 Acked 1024 Read 4 Sent Done Done false
175162024-02-03T17:30:17.060Z 26 Acked 1025 Read 2 Sent Done Done false
175172024-02-03T17:30:17.060Z 27 Acked 1026 Read 3 Sent Done Done false
175182024-02-03T17:30:17.061Z 28 Acked 1027 Write 5 Sent Done Done false
175192024-02-03T17:30:17.061Z 29 Acked 1028 Write 2 Sent Done Done false
175202024-02-03T17:30:17.061Z 30 Acked 1029 Read 1 Sent Done Sent false
175212024-02-03T17:30:17.061Z STATES DS:0 DS:1 DS:2 TOTAL
175222024-02-03T17:30:17.061Z New 0 0 0 0
175232024-02-03T17:30:17.061Z Sent 28 0 1 29
175242024-02-03T17:30:17.061Z Done 2 30 29 61
175252024-02-03T17:30:17.061Z Skipped 0 0 0 0
175262024-02-03T17:30:17.061Z Error 0 0 0 0
175272024-02-03T17:30:17.061ZLast Flush: 0 1005 1005
175282024-02-03T17:30:17.061ZDownstairs last five completed:
175292024-02-03T17:30:17.061ZUpstairs last five completed: 30 29 28 27 26
175302024-02-03T17:30:17.065Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:16.232227233Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1783}
175312024-02-03T17:30:17.065ZWrote out file "/tmp/test_repair_verify.data"
175322024-02-03T17:30:17.065Z
175332024-02-03T17:30:17.073ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175342024-02-03T17:30:17.104ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175352024-02-03T17:30:17.104Z 0 000-019 161 161 161 296 296 296 F F F
175362024-02-03T17:30:17.104Z 1 020-039 160 160 160 293 293 293 F F F
175372024-02-03T17:30:17.105Z 2 040-059 155 155 155 284 284 284 F F F
175382024-02-03T17:30:17.105Z 3 060-079 157 157 157 288 288 288 F F F
175392024-02-03T17:30:17.109Z 4 080-099 156 156 156 287 287 287 F T T <---
175402024-02-03T17:30:17.109Z 5 100-119 161 161 161 296 296 296 F T T <---
175412024-02-03T17:30:17.109Z 6 120-139 154 154 154 282 282 282 F F F
175422024-02-03T17:30:17.109Z 7 140-159 159 159 159 292 292 292 F F F
175432024-02-03T17:30:17.109Z 8 160-179 161 161 161 296 296 296 F T T <---
175442024-02-03T17:30:17.109Z 9 180-199 159 159 159 292 292 292 F T T <---
175452024-02-03T17:30:17.109Z 10 200-219 156 156 156 285 285 285 F T T <---
175462024-02-03T17:30:17.109Z 11 220-239 158 158 158 291 291 291 F T T <---
175472024-02-03T17:30:17.109Z 12 240-259 160 160 160 295 295 295 F T T <---
175482024-02-03T17:30:17.109Z 13 260-279 160 160 160 295 295 295 F T T <---
175492024-02-03T17:30:17.109Z 14 280-299 155 155 155 284 284 284 F F F
175502024-02-03T17:30:17.110Z 15 300-319 151 151 151 279 279 279 F F F
175512024-02-03T17:30:17.110Z 16 320-339 159 159 159 292 292 292 F F F
175522024-02-03T17:30:17.110Z 17 340-359 160 160 160 294 294 294 F F F
175532024-02-03T17:30:17.110Z 18 360-379 161 162 162 296 297 297 T F F <---
175542024-02-03T17:30:17.110Z 19 380-399 158 158 158 291 291 291 F F F
175552024-02-03T17:30:17.110Z 20 400-419 156 162 162 285 297 297 T F F <---
175562024-02-03T17:30:17.110Z 21 420-439 161 161 161 296 296 296 F F F
175572024-02-03T17:30:17.110Z 22 440-459 154 154 154 282 282 282 F F F
175582024-02-03T17:30:17.110Z 23 460-479 158 158 158 291 291 291 F F F
175592024-02-03T17:30:17.110Z 24 480-499 156 156 156 285 285 285 F T T <---
175602024-02-03T17:30:17.110Z 25 500-519 161 161 161 296 296 296 F T T <---
175612024-02-03T17:30:17.111Z 26 520-539 161 161 161 296 296 296 F T T <---
175622024-02-03T17:30:17.111Z 27 540-559 161 161 161 296 296 296 F F F
175632024-02-03T17:30:17.111Z 28 560-579 156 156 156 285 285 285 F T T <---
175642024-02-03T17:30:17.111Z 29 580-599 160 160 160 293 293 293 F F F
175652024-02-03T17:30:17.111ZMax gen: 162, Max flush: 297
175662024-02-03T17:30:17.111ZError: Difference in extent metadata found!
175672024-02-03T17:30:17.126ZA Difference in extent metadata is expected here
175682024-02-03T17:30:17.126ZOn loop 81
175692024-02-03T17:30:17.126Z
175702024-02-03T17:30:17.126ZVerifying data now
175712024-02-03T17:30:23.351ZNo change after verify
175722024-02-03T17:30:23.351ZLoop: 81 Downstairs dump after verify (and repair):
175732024-02-03T17:30:23.375ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175742024-02-03T17:30:23.375Z 0 000-019 161 161 161 296 296 296 F F F
175752024-02-03T17:30:23.375Z 1 020-039 160 160 160 293 293 293 F F F
175762024-02-03T17:30:23.375Z 2 040-059 155 155 155 284 284 284 F F F
175772024-02-03T17:30:23.376Z 3 060-079 157 157 157 288 288 288 F F F
175782024-02-03T17:30:23.376Z 4 080-099 163 163 163 298 298 298 F F F
175792024-02-03T17:30:23.376Z 5 100-119 163 163 163 298 298 298 F F F
175802024-02-03T17:30:23.376Z 6 120-139 154 154 154 282 282 282 F F F
175812024-02-03T17:30:23.376Z 7 140-159 159 159 159 292 292 292 F F F
175822024-02-03T17:30:23.376Z 8 160-179 163 163 163 298 298 298 F F F
175832024-02-03T17:30:23.376Z 9 180-199 163 163 163 298 298 298 F F F
175842024-02-03T17:30:23.376Z 10 200-219 163 163 163 298 298 298 F F F
175852024-02-03T17:30:23.376Z 11 220-239 163 163 163 298 298 298 F F F
175862024-02-03T17:30:23.376Z 12 240-259 163 163 163 298 298 298 F F F
175872024-02-03T17:30:23.376Z 13 260-279 163 163 163 298 298 298 F F F
175882024-02-03T17:30:23.376Z 14 280-299 155 155 155 284 284 284 F F F
175892024-02-03T17:30:23.376Z 15 300-319 151 151 151 279 279 279 F F F
175902024-02-03T17:30:23.376Z 16 320-339 159 159 159 292 292 292 F F F
175912024-02-03T17:30:23.376Z 17 340-359 160 160 160 294 294 294 F F F
175922024-02-03T17:30:23.380Z 18 360-379 162 162 162 297 297 297 F F F
175932024-02-03T17:30:23.380Z 19 380-399 158 158 158 291 291 291 F F F
175942024-02-03T17:30:23.380Z 20 400-419 162 162 162 297 297 297 F F F
175952024-02-03T17:30:23.380Z 21 420-439 161 161 161 296 296 296 F F F
175962024-02-03T17:30:23.380Z 22 440-459 154 154 154 282 282 282 F F F
175972024-02-03T17:30:23.380Z 23 460-479 158 158 158 291 291 291 F F F
175982024-02-03T17:30:23.380Z 24 480-499 163 163 163 298 298 298 F F F
175992024-02-03T17:30:23.380Z 25 500-519 163 163 163 298 298 298 F F F
176002024-02-03T17:30:23.380Z 26 520-539 163 163 163 298 298 298 F F F
176012024-02-03T17:30:23.380Z 27 540-559 161 161 161 296 296 296 F F F
176022024-02-03T17:30:23.380Z 28 560-579 163 163 163 298 298 298 F F F
176032024-02-03T17:30:23.380Z 29 580-599 160 160 160 293 293 293 F F F
176042024-02-03T17:30:23.380ZMax gen: 163, Max flush: 298
176052024-02-03T17:30:23.383Z
176062024-02-03T17:30:23.383ZBegin loop 82
176072024-02-03T17:30:23.384ZDownstairs to restart: 2
176082024-02-03T17:30:23.414Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.589682848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791}
176092024-02-03T17:30:23.414Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.590162682Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791}
176102024-02-03T17:30:23.419Z{Crucible runtime is spawned
176112024-02-03T17:30:23.419Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.590404023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791}
176122024-02-03T17:30:23.419Z{"msg":"Crucible f17dde6f-3bec-4034-b78c-c5534f2a7822 has session id: dfd990af-e4d3-4172-b1ff-185be6fe549c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.590528593Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176132024-02-03T17:30:23.419Z{"msg":"Upstairs opts: Upstairs UUID: f17dde6f-3bec-4034-b78c-c5534f2a7822, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.590686767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176142024-02-03T17:30:23.420Z{"msg":"Crucible stats registered with UUID: f17dde6f-3bec-4034-b78c-c5534f2a7822","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.590844859Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176152024-02-03T17:30:23.420Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.590887364Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791}
176162024-02-03T17:30:23.420Z{"msg":"f17dde6f-3bec-4034-b78c-c5534f2a7822 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.590980023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176172024-02-03T17:30:23.420Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591047414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176182024-02-03T17:30:23.420Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591093924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176192024-02-03T17:30:23.420Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591316689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176202024-02-03T17:30:23.420Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591387385Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176212024-02-03T17:30:23.420Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.59142398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176222024-02-03T17:30:23.420Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591466277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176232024-02-03T17:30:23.420Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591505582Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176242024-02-03T17:30:23.420Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591544032Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176252024-02-03T17:30:23.420Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591604182Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176262024-02-03T17:30:23.420Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591631735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176272024-02-03T17:30:23.420Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591686293Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176282024-02-03T17:30:23.420Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.591756625Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176292024-02-03T17:30:23.420Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.592623804Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176302024-02-03T17:30:23.420Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.592853148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176312024-02-03T17:30:23.420Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.592975455Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176322024-02-03T17:30:23.420Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.593022766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176332024-02-03T17:30:23.420Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.593547651Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176342024-02-03T17:30:23.420Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.593758771Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176352024-02-03T17:30:23.420Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.593844679Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176362024-02-03T17:30:23.420Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.5939487Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176372024-02-03T17:30:23.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.594527462Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176382024-02-03T17:30:23.425Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.594642463Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176392024-02-03T17:30:23.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.594726419Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176402024-02-03T17:30:23.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.594911851Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176412024-02-03T17:30:23.425Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.594992274Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176422024-02-03T17:30:23.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.595085096Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176432024-02-03T17:30:23.425Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.5952874Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176442024-02-03T17:30:23.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.595989574Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176452024-02-03T17:30:23.425Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.59607116Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176462024-02-03T17:30:23.426Z{"msg":"[0]R flush_numbers[0..12]: [296, 293, 284, 288, 298, 298, 282, 292, 298, 298, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596118401Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176472024-02-03T17:30:23.426Z{"msg":"[0]R generation[0..12]: [161, 160, 155, 157, 163, 163, 154, 159, 163, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596190373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176482024-02-03T17:30:23.426Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596261991Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176492024-02-03T17:30:23.426Z{"msg":"[1]R flush_numbers[0..12]: [296, 293, 284, 288, 298, 298, 282, 292, 298, 298, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596332573Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176502024-02-03T17:30:23.426Z{"msg":"[1]R generation[0..12]: [161, 160, 155, 157, 163, 163, 154, 159, 163, 163, 163, 163]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
176512024-02-03T17:30:23.426Z,"time":"2024-02-03T17:30:22.596403987Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176522024-02-03T17:30:23.426Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596483506Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176532024-02-03T17:30:23.426Z{"msg":"[2]R flush_numbers[0..12]: [296, 293, 284, 288, 298, 298, 282, 292, 298, 298, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596557699Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176542024-02-03T17:30:23.426Z{"msg":"[2]R generation[0..12]: [161, 160, 155, 157, 163, 163, 154, 159, 163, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596636894Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176552024-02-03T17:30:23.426Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596714916Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176562024-02-03T17:30:23.426Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596791237Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176572024-02-03T17:30:23.426Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596836062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176582024-02-03T17:30:23.426Z{"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596859114Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176592024-02-03T17:30:23.426Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596880887Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176602024-02-03T17:30:23.426Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596933449Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176612024-02-03T17:30:23.426Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.596971448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176622024-02-03T17:30:23.426Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.597036451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176632024-02-03T17:30:23.426Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.597110461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176642024-02-03T17:30:23.426Z}
176652024-02-03T17:30:23.426Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.597149286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176662024-02-03T17:30:23.426Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.597207448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176672024-02-03T17:30:23.426Z{"msg":"f17dde6f-3bec-4034-b78c-c5534f2a7822 is now active with session: dfd990af-e4d3-4172-b1ff-185be6fe549c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.597234696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176682024-02-03T17:30:23.426Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.59725465Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791,"session_id":"dfd990af-e4d3-4172-b1ff-185be6fe549c"}
176692024-02-03T17:30:23.426Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:22.597295053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791}
176702024-02-03T17:30:23.427ZLoading write count information from file "/tmp/test_repair_verify.data"
176712024-02-03T17:30:23.427ZRun Repair workload
176722024-02-03T17:30:23.427Z01/30 Write block 370 len 16384 data: 10 12 10 9
176732024-02-03T17:30:23.430Z02/30 Write block 241 len 40960 data: 10 7 5 7 8 9 11 12 13 18
176742024-02-03T17:30:23.430Z03/30 Read block 549 len 36864
176752024-02-03T17:30:23.430Z04/30 Flush
176762024-02-03T17:30:23.435Z05/30 Flush
176772024-02-03T17:30:23.435Z06/30 Write block 552 len 28672 data: 12 13 12 11 13 12 13
176782024-02-03T17:30:23.438Z07/30 Write block 98 len 40960 data: 9 12 14 14 14 17 15 18 16 17
176792024-02-03T17:30:23.438Z08/30 Write block 488 len 12288 data: 11 13 13
176802024-02-03T17:30:23.438Z09/30 Flush
176812024-02-03T17:30:23.442Z10/30 Write block 513 len 28672 data: 12 9 11 9 12 11 8
176822024-02-03T17:30:23.442Z11/30 Write block 513 len 40960 data: 13 10 12 10 13 12 9 8 8 7
176832024-02-03T17:30:23.445Z12/30 Read block 506 len 8192
176842024-02-03T17:30:23.445Z13/30 Read block 450 len 8192
176852024-02-03T17:30:23.445Z14/30 Read block 13 len 28672
176862024-02-03T17:30:23.448Z15/30 Read block 209 len 32768
176872024-02-03T17:30:23.448Z16/30 Read block 511 len 4096
176882024-02-03T17:30:23.454Z17/30 Read block 288 len 20480
176892024-02-03T17:30:23.454Z18/30 Write block 82 len 12288 data: 11 14 15
176902024-02-03T17:30:23.454Z19/30 Read block 162 len 16384
176912024-02-03T17:30:23.458Z20/30 Read block 215 len 8192
176922024-02-03T17:30:23.458Z21/30 Flush
176932024-02-03T17:30:23.458Z22/30 Write block 56 len 20480 data: 14 13 16 16 15
176942024-02-03T17:30:23.458Z23/30 Write block 536 len 16384 data: 11 11 10 10
176952024-02-03T17:30:23.461Z24/30 Write block 471 len 24576 data: 11 9 8 10 9 10
176962024-02-03T17:30:23.461Z25/30 Write block 559 len 32768 data: 14 10 9 6 8 9 11 13
176972024-02-03T17:30:23.461Z26/30 Read block 50 len 4096
176982024-02-03T17:30:23.464Z27/30 Read block 545 len 40960
176992024-02-03T17:30:23.467Z28/30 Write block 554 len 36864 data: 13 12 14 13 14 15 11 10 7
177002024-02-03T17:30:23.467Z29/30 Read block 140 len 36864
177012024-02-03T17:30:23.472Z30/30 Write block 369 len 32768 data: 11 11 13 11 10 10 10 13
177022024-02-03T17:30:23.472Z----------------------------------------------------------------
177032024-02-03T17:30:23.472Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
177042024-02-03T17:30:23.472ZStates: Active Active Active
177052024-02-03T17:30:23.472ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177062024-02-03T17:30:23.472Z 1 Acked 1000 Write 4 Done Done Sent false
177072024-02-03T17:30:23.472Z 2 Acked 1001 Write 10 Done Done Sent false
177082024-02-03T17:30:23.472Z 3 Acked 1002 Read 9 Done Done Sent false
177092024-02-03T17:30:23.472Z 4 Acked 1003 Flush 0 Done Done Sent false
177102024-02-03T17:30:23.472Z 5 Acked 1004 Flush 0 Done Done Sent false
177112024-02-03T17:30:23.472Z 6 Acked 1005 Write 7 Done Done Sent false
177122024-02-03T17:30:23.472Z 7 Acked 1006 Write 10 Done Done Sent false
177132024-02-03T17:30:23.472Z 8 Acked 1007 Write 3 Done Done Sent false
177142024-02-03T17:30:23.472Z 9 Acked 1008 Flush 0 Done Done Sent false
177152024-02-03T17:30:23.472Z 10 Acked 1009 Write 7 Done Done Sent false
177162024-02-03T17:30:23.472Z 11 Acked 1010 Write 10 Done Done Sent false
177172024-02-03T17:30:23.472Z 12 Acked 1011 Read 2 Done Done Sent false
177182024-02-03T17:30:23.473Z 13 Acked 1012 Read 2 Done Done Sent false
177192024-02-03T17:30:23.473Z 14 Acked 1013 Read 7 Done Done Sent false
177202024-02-03T17:30:23.473Z 15 Acked 1014 Read 8 Done Done Sent false
177212024-02-03T17:30:23.473Z 16 Acked 1015 Read 1 Done Done Sent false
177222024-02-03T17:30:23.473Z 17 Acked 1016 Read 5 Done Done Sent false
177232024-02-03T17:30:23.473Z 18 Acked 1017 Write 3 Done Done Sent false
177242024-02-03T17:30:23.473Z 19 Acked 1018 Read 4 Done Done Sent false
177252024-02-03T17:30:23.473Z 20 Acked 1019 Read 2 Done Done Sent false
177262024-02-03T17:30:23.473Z 21 Acked 1020 Flush 0 Done Done Sent false
177272024-02-03T17:30:23.473Z 22 Acked 1021 Write 5 Done Done Sent false
177282024-02-03T17:30:23.473Z 23 Acked 1022 Write 4 Done Done Sent false
177292024-02-03T17:30:23.473Z 24 Acked 1023 Write 6 Done Done Sent false
177302024-02-03T17:30:23.473Z 25 Acked 1024 Write 8 Done Done Sent false
177312024-02-03T17:30:23.473Z 26 Acked 1025 Read 1 Done Done Sent false
177322024-02-03T17:30:23.473Z 27 Acked 1026 Read 10 Done Done Sent false
177332024-02-03T17:30:23.473Z 28 Acked 1027 Write 9 Done Done Sent false
177342024-02-03T17:30:23.473Z 29 Acked 1028 Read 9 Sent Done Sent false
177352024-02-03T17:30:23.473Z 30 Acked 1029 Write 8 Sent Sent Sent false
177362024-02-03T17:30:23.473Z STATES DS:0 DS:1 DS:2 TOTAL
177372024-02-03T17:30:23.473Z New 0 0 0 0
177382024-02-03T17:30:23.473Z Sent 2 1 30 33
177392024-02-03T17:30:23.473Z Done 28 29 0 57
177402024-02-03T17:30:23.473Z Skipped 0 0 0 0
177412024-02-03T17:30:23.473Z Error 0 0 0 0
177422024-02-03T17:30:23.474ZLast Flush: 1020 1020 0
177432024-02-03T17:30:23.474ZDownstairs last five completed:
177442024-02-03T17:30:23.477ZUpstairs last five completed: 30 29 28 27 26
177452024-02-03T17:30:23.477Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:22.646654217Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1791}
177462024-02-03T17:30:23.477ZWrote out file "/tmp/test_repair_verify.data"
177472024-02-03T17:30:23.483Z
177482024-02-03T17:30:23.486ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177492024-02-03T17:30:23.514ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177502024-02-03T17:30:23.514Z 0 000-019 161 161 161 296 296 296 F F F
177512024-02-03T17:30:23.518Z 1 020-039 160 160 160 293 293 293 F F F
177522024-02-03T17:30:23.518Z 2 040-059 155 155 155 284 284 284 T T F <---
177532024-02-03T17:30:23.518Z 3 060-079 157 157 157 288 288 288 T T F <---
177542024-02-03T17:30:23.518Z 4 080-099 164 164 163 302 302 298 F F F <---
177552024-02-03T17:30:23.518Z 5 100-119 164 164 163 301 301 298 F F F <---
177562024-02-03T17:30:23.518Z 6 120-139 154 154 154 282 282 282 F F F
177572024-02-03T17:30:23.518Z 7 140-159 159 159 159 292 292 292 F F F
177582024-02-03T17:30:23.518Z 8 160-179 163 163 163 298 298 298 F F F
177592024-02-03T17:30:23.518Z 9 180-199 163 163 163 298 298 298 F F F
177602024-02-03T17:30:23.518Z 10 200-219 163 163 163 298 298 298 F F F
177612024-02-03T17:30:23.518Z 11 220-239 163 163 163 298 298 298 F F F
177622024-02-03T17:30:23.518Z 12 240-259 164 164 163 299 299 298 F F F <---
177632024-02-03T17:30:23.518Z 13 260-279 163 163 163 298 298 298 F F F
177642024-02-03T17:30:23.518Z 14 280-299 155 155 155 284 284 284 F F F
177652024-02-03T17:30:23.518Z 15 300-319 151 151 151 279 279 279 F F F
177662024-02-03T17:30:23.518Z 16 320-339 159 159 159 292 292 292 F F F
177672024-02-03T17:30:23.518Z 17 340-359 160 160 160 294 294 294 F F F
177682024-02-03T17:30:23.518Z 18 360-379 164 164 162 299 299 297 T T F <---
177692024-02-03T17:30:23.518Z 19 380-399 158 158 158 291 291 291 F F F
177702024-02-03T17:30:23.518Z 20 400-419 162 162 162 297 297 297 F F F
177712024-02-03T17:30:23.519Z 21 420-439 161 161 161 296 296 296 F F F
177722024-02-03T17:30:23.519Z 22 440-459 154 154 154 282 282 282 F F F
177732024-02-03T17:30:23.519Z 23 460-479 158 158 158 291 291 291 T T F <---
177742024-02-03T17:30:23.519Z 24 480-499 164 164 163 301 301 298 F F F <---
177752024-02-03T17:30:23.519Z 25 500-519 164 164 163 302 302 298 F F F <---
177762024-02-03T17:30:23.519Z 26 520-539 164 164 163 302 302 298 T T F <---
177772024-02-03T17:30:23.519Z 27 540-559 164 164 161 301 301 296 T T F <---
177782024-02-03T17:30:23.519Z 28 560-579 163 163 163 298 298 298 T T F <---
177792024-02-03T17:30:23.519Z 29 580-599 160 160 160 293 293 293 F F F
177802024-02-03T17:30:23.519ZMax gen: 164, Max flush: 302
177812024-02-03T17:30:23.519ZError: Difference in extent metadata found!
177822024-02-03T17:30:23.522ZA Difference in extent metadata is expected here
177832024-02-03T17:30:23.522ZOn loop 82
177842024-02-03T17:30:23.522Z
177852024-02-03T17:30:23.526ZVerifying data now
177862024-02-03T17:30:28.605ZNo change after verify
177872024-02-03T17:30:28.605ZLoop: 82 Downstairs dump after verify (and repair):
177882024-02-03T17:30:28.632ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177892024-02-03T17:30:28.633Z 0 000-019 161 161 161 296 296 296 F F F
177902024-02-03T17:30:28.633Z 1 020-039 160 160 160 293 293 293 F F F
177912024-02-03T17:30:28.633Z 2 040-059 165 165 165 303 303 303 F F F
177922024-02-03T17:30:28.633Z 3 060-079 165 165 165 303 303 303 F F F
177932024-02-03T17:30:28.633Z 4 080-099 164 164 164 302 302 302 F F F
177942024-02-03T17:30:28.633Z 5 100-119 164 164 164 301 301 301 F F F
177952024-02-03T17:30:28.633Z 6 120-139 154 154 154 282 282 282 F F F
177962024-02-03T17:30:28.638Z 7 140-159 159 159 159 292 292 292 F F F
177972024-02-03T17:30:28.638Z 8 160-179 163 163 163 298 298 298 F F F
177982024-02-03T17:30:28.638Z 9 180-199 163 163 163 298 298 298 F F F
177992024-02-03T17:30:28.638Z 10 200-219 163 163 163 298 298 298 F F F
178002024-02-03T17:30:28.638Z 11 220-239 163 163 163 298 298 298 F F F
178012024-02-03T17:30:28.638Z 12 240-259 164 164 164 299 299 299 F F F
178022024-02-03T17:30:28.639Z 13 260-279 163 163 163 298 298 298 F F F
178032024-02-03T17:30:28.639Z 14 280-299 155 155 155 284 284 284 F F F
178042024-02-03T17:30:28.639Z 15 300-319 151 151 151 279 279 279 F F F
178052024-02-03T17:30:28.639Z 16 320-339 159 159 159 292 292 292 F F F
178062024-02-03T17:30:28.639Z 17 340-359 160 160 160 294 294 294 F F F
178072024-02-03T17:30:28.639Z 18 360-379 165 165 165 303 303 303 F F F
178082024-02-03T17:30:28.639Z 19 380-399 158 158 158 291 291 291 F F F
178092024-02-03T17:30:28.639Z 20 400-419 162 162 162 297 297 297 F F F
178102024-02-03T17:30:28.639Z 21 420-439 161 161 161 296 296 296 F F F
178112024-02-03T17:30:28.639Z 22 440-459 154 154 154 282 282 282 F F F
178122024-02-03T17:30:28.639Z 23 460-479 165 165 165 303 303 303 F F F
178132024-02-03T17:30:28.639Z 24 480-499 164 164 164 301 301 301 F F F
178142024-02-03T17:30:28.639Z 25 500-519 164 164 164 302 302 302 F F F
178152024-02-03T17:30:28.639Z 26 520-539 165 165 165 303 303 303 F F F
178162024-02-03T17:30:28.639Z 27 540-559 165 165 165 303 303 303 F F F
178172024-02-03T17:30:28.639Z 28 560-579 165 165 165 303 303 303 F F F
178182024-02-03T17:30:28.639Z 29 580-599 160 160 160 293 293 293 F F F
178192024-02-03T17:30:28.639ZMax gen: 165, Max flush: 303
178202024-02-03T17:30:28.639Z
178212024-02-03T17:30:28.639ZBegin loop 83
178222024-02-03T17:30:28.639ZDownstairs to restart: 1
178232024-02-03T17:30:28.669Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.844524246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799}
178242024-02-03T17:30:28.669Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845046142Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799}
178252024-02-03T17:30:28.669Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845120586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799}
178262024-02-03T17:30:28.669Z{"msg":"Crucible e530e2bb-72d3-41fe-a79b-9885cb6cabec has session id: bfc5e697-0822-48e2-b435-3753a24f2260","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
178272024-02-03T17:30:28.669Z2024-02-03T17:30:27.845163048Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178282024-02-03T17:30:28.673Z{"msg":"Upstairs opts: Upstairs UUID: e530e2bb-72d3-41fe-a79b-9885cb6cabec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845253112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178292024-02-03T17:30:28.673Z{"msg":"Crucible stats registered with UUID: e530e2bb-72d3-41fe-a79b-9885cb6cabec","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845340847Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178302024-02-03T17:30:28.674Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845378732Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799}
178312024-02-03T17:30:28.674Z{"msg":"e530e2bb-72d3-41fe-a79b-9885cb6cabec active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845439969Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178322024-02-03T17:30:28.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845517414Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178332024-02-03T17:30:28.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845546671Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178342024-02-03T17:30:28.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845579546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178352024-02-03T17:30:28.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845603288Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178362024-02-03T17:30:28.674Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.84564052Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178372024-02-03T17:30:28.674Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845682149Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178382024-02-03T17:30:28.674Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.84575179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178392024-02-03T17:30:28.674Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.84584387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178402024-02-03T17:30:28.674Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845940711Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178412024-02-03T17:30:28.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.845991199Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178422024-02-03T17:30:28.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.846102784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178432024-02-03T17:30:28.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.846310111Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178442024-02-03T17:30:28.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.847310418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178452024-02-03T17:30:28.674Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.847381082Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178462024-02-03T17:30:28.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.847408177Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178472024-02-03T17:30:28.674Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.847431268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178482024-02-03T17:30:28.674Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.84841532Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178492024-02-03T17:30:28.674Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.848508203Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178502024-02-03T17:30:28.681Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.849203498Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178512024-02-03T17:30:28.681Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.849294344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178522024-02-03T17:30:28.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.849552337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178532024-02-03T17:30:28.681Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.849633404Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178542024-02-03T17:30:28.681Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.849706198Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178552024-02-03T17:30:28.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.849775418Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178562024-02-03T17:30:28.681Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.849824913Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178572024-02-03T17:30:28.681Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.849849036Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178582024-02-03T17:30:28.681Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.850199753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178592024-02-03T17:30:28.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.850950019Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178602024-02-03T17:30:28.681Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851048684Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178612024-02-03T17:30:28.681Z{"msg":"[0]R flush_numbers[0..12]: [296, 293, 303, 303, 302, 301, 282, 292, 298, 298, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851077236Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178622024-02-03T17:30:28.682Z{"msg":"[0]R generation[0..12]: [161, 160, 165, 165, 164, 164, 154, 159, 163, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851121407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178632024-02-03T17:30:28.682Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851147423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178642024-02-03T17:30:28.682Z{"msg":"[1]R flush_numbers[0..12]: [296, 293, 303, 303, 302, 301, 282, 292, 298, 298, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851190354Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178652024-02-03T17:30:28.682Z{"msg":"[1]R generation[0..12]: [161, 160, 165, 165, 164, 164, 154, 159, 163, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851215412Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178662024-02-03T17:30:28.682Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851257747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178672024-02-03T17:30:28.682Z{"msg":"[2]R flush_numbers[0..12]: [296, 293, 303, 303, 302, 301, 282, 292, 298, 298, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851283329Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178682024-02-03T17:30:28.682Z{"msg":"[2]R generation[0..12]: [161, 160, 165, 165, 164, 164, 154, 159, 163, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851338817Z","hostname":"Wait for a query_work_queue command to finish before sending IO
178692024-02-03T17:30:28.682Zip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178702024-02-03T17:30:28.682Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851477508Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178712024-02-03T17:30:28.682Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851533533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178722024-02-03T17:30:28.682Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851559496Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178732024-02-03T17:30:28.682Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851581492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178742024-02-03T17:30:28.682Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851633973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178752024-02-03T17:30:28.682Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851702469Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178762024-02-03T17:30:28.682Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851729305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178772024-02-03T17:30:28.683Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851791068Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178782024-02-03T17:30:28.683Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851859494Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178792024-02-03T17:30:28.683Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851887507Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178802024-02-03T17:30:28.683Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851929126Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178812024-02-03T17:30:28.683Z{"msg":"e530e2bb-72d3-41fe-a79b-9885cb6cabec is now active with session: bfc5e697-0822-48e2-b435-3753a24f2260","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.851953229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178822024-02-03T17:30:28.683Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.852008438Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799,"session_id":"bfc5e697-0822-48e2-b435-3753a24f2260"}
178832024-02-03T17:30:28.683Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:27.852075789Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799}
178842024-02-03T17:30:28.683ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178852024-02-03T17:30:28.683ZLoading write count information from file "/tmp/test_repair_verify.data"
178862024-02-03T17:30:28.683ZRun Repair workload
178872024-02-03T17:30:28.683Z01/30 Flush
178882024-02-03T17:30:28.683Z02/30 Write block 468 len 16384 data: 11 11 11 12
178892024-02-03T17:30:28.688Z03/30 Flush
178902024-02-03T17:30:28.689Z04/30 Write block 266 len 40960 data: 16 13 15 17 19 18 16 16 19 21
178912024-02-03T17:30:28.689Z05/30 Read block 186 len 36864
178922024-02-03T17:30:28.692Z06/30 Read block 214 len 20480
178932024-02-03T17:30:28.692Z07/30 Write block 578 len 12288 data: 17 17 18
178942024-02-03T17:30:28.692Z08/30 Read block 535 len 28672
178952024-02-03T17:30:28.697Z09/30 Read block 353 len 28672
178962024-02-03T17:30:28.697Z10/30 Read block 477 len 24576
178972024-02-03T17:30:28.697Z11/30 Read block 327 len 32768
178982024-02-03T17:30:28.700Z12/30 Read block 190 len 16384
178992024-02-03T17:30:28.701Z13/30 Write block 444 len 24576 data: 11 13 12 10 11 8
179002024-02-03T17:30:28.701Z14/30 Write block 90 len 36864 data: 11 10 11 11 11 10 11 12 10
179012024-02-03T17:30:28.704Z15/30 Write block 548 len 24576 data: 18 19 16 13 13 14
179022024-02-03T17:30:28.704Z16/30 Write block 61 len 8192 data: 15 14
179032024-02-03T17:30:28.704Z17/30 Write block 80 len 12288 data: 10 9 12
179042024-02-03T17:30:28.704Z18/30 Flush
179052024-02-03T17:30:28.708Z19/30 Flush
179062024-02-03T17:30:28.712Z20/30 Write block 21 len 16384 data: 7 9 9 13
179072024-02-03T17:30:28.712Z21/30 Write block 226 len 12288 data: 14 20 18
179082024-02-03T17:30:28.712Z22/30 Read block 376 len 4096
179092024-02-03T17:30:28.715Z23/30 Write block 571 len 36864 data: 11 7 8 8 10 12 14 18 18
179102024-02-03T17:30:28.715Z24/30 Read block 507 len 28672
179112024-02-03T17:30:28.715Z25/30 Read block 381 len 8192
179122024-02-03T17:30:28.719Z26/30 Write block 507 len 24576 data: 23 19 19 15 14 14
179132024-02-03T17:30:28.719Z27/30 Read block 465 len 28672
179142024-02-03T17:30:28.723Z28/30 Write block 109 len 20480 data: 16 16 14 13 16
179152024-02-03T17:30:28.723Z29/30 Write block 38 len 40960 data: 11 12 10 8 10 10 9 7 6 6
179162024-02-03T17:30:28.723Z30/30 Write block 342 len 12288 data: 10 11 11
179172024-02-03T17:30:28.727Z----------------------------------------------------------------
179182024-02-03T17:30:28.727Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:27
179192024-02-03T17:30:28.727ZStates: Active Active Active
179202024-02-03T17:30:28.727ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179212024-02-03T17:30:28.727Z 4 Acked 1003 Write 10 Done Done Done false
179222024-02-03T17:30:28.727Z 5 Acked 1004 Read 9 Done Done Done false
179232024-02-03T17:30:28.727Z 6 Acked 1005 Read 5 Done Sent Done false
179242024-02-03T17:30:28.727Z 7 Acked 1006 Write 3 Done Sent Done false
179252024-02-03T17:30:28.727Z 8 Acked 1007 Read 7 Done Sent Done false
179262024-02-03T17:30:28.727Z 9 Acked 1008 Read 7 Done Sent Done false
179272024-02-03T17:30:28.727Z 10 Acked 1009 Read 6 Done Sent Done false
179282024-02-03T17:30:28.728Z 11 Acked 1010 Read 8 Done Sent Done false
179292024-02-03T17:30:28.728Z 12 Acked 1011 Read 4 Done Sent Done false
179302024-02-03T17:30:28.728Z 13 Acked 1012 Write 6 Done Sent Done false
179312024-02-03T17:30:28.728Z 14 Acked 1013 Write 9 Done Sent Done false
179322024-02-03T17:30:28.728Z 15 Acked 1014 Write 6 Done Sent Done false
179332024-02-03T17:30:28.728Z 16 Acked 1015 Write 2 Done Sent Done false
179342024-02-03T17:30:28.728Z 17 Acked 1016 Write 3 Done Sent Done false
179352024-02-03T17:30:28.728Z 18 Acked 1017 Flush 0 Done Sent Done false
179362024-02-03T17:30:28.728Z 19 Acked 1018 Flush 0 Done Sent Done false
179372024-02-03T17:30:28.728Z 20 Acked 1019 Write 4 Done Sent Done false
179382024-02-03T17:30:28.728Z 21 Acked 1020 Write 3 Done Sent Done false
179392024-02-03T17:30:28.728Z 22 Acked 1021 Read 1 Done Sent Done false
179402024-02-03T17:30:28.728Z 23 Acked 1022 Write 9 Done Sent Done false
179412024-02-03T17:30:28.729Z 24 Acked 1023 Read 7 Done Sent Done false
179422024-02-03T17:30:28.729Z 25 Acked 1024 Read 2 Done Sent Done false
179432024-02-03T17:30:28.729Z 26 Acked 1025 Write 6 Done Sent Done false
179442024-02-03T17:30:28.729Z 27 Acked 1026 Read 7 Done Sent Done false
179452024-02-03T17:30:28.729Z 28 Acked 1027 Write 5 Done Sent Done false
179462024-02-03T17:30:28.729Z 29 Acked 1028 Write 10 Sent Sent Sent false
179472024-02-03T17:30:28.729Z 30 Acked 1029 Write 3 Sent Sent Sent false
179482024-02-03T17:30:28.729Z STATES DS:0 DS:1 DS:2 TOTAL
179492024-02-03T17:30:28.729Z New 0 0 0 0
179502024-02-03T17:30:28.729Z Sent 2 25 2 29
179512024-02-03T17:30:28.729Z Done 25 2 25 52
179522024-02-03T17:30:28.729Z Skipped 0 0 0 0
179532024-02-03T17:30:28.729Z Error 0 0 0 0
179542024-02-03T17:30:28.729ZLast Flush: 1018 1002 1018
179552024-02-03T17:30:28.729ZDownstairs last five completed: 1002 1001 1000
179562024-02-03T17:30:28.729ZUpstairs last five completed: 30 29 28 27 26
179572024-02-03T17:30:28.729Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:27.899010896Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1799}
179582024-02-03T17:30:28.730ZWrote out file "/tmp/test_repair_verify.data"
179592024-02-03T17:30:28.735Z
179602024-02-03T17:30:28.740ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179612024-02-03T17:30:28.769ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179622024-02-03T17:30:28.769Z 0 000-019 161 161 161 296 296 296 F F F
179632024-02-03T17:30:28.769Z 1 020-039 160 160 160 293 293 293 T F T <---
179642024-02-03T17:30:28.770Z 2 040-059 165 165 165 303 303 303 T F T <---
179652024-02-03T17:30:28.774Z 3 060-079 166 165 166 306 303 306 F F F <---
179662024-02-03T17:30:28.774Z 4 080-099 166 164 166 306 302 306 F F F <---
179672024-02-03T17:30:28.774Z 5 100-119 164 164 164 301 301 301 T F T <---
179682024-02-03T17:30:28.774Z 6 120-139 154 154 154 282 282 282 F F F
179692024-02-03T17:30:28.774Z 7 140-159 159 159 159 292 292 292 F F F
179702024-02-03T17:30:28.774Z 8 160-179 163 163 163 298 298 298 F F F
179712024-02-03T17:30:28.774Z 9 180-199 163 163 163 298 298 298 F F F
179722024-02-03T17:30:28.774Z 10 200-219 163 163 163 298 298 298 F F F
179732024-02-03T17:30:28.774Z 11 220-239 163 163 163 298 298 298 T F T <---
179742024-02-03T17:30:28.774Z 12 240-259 164 164 164 299 299 299 F F F
179752024-02-03T17:30:28.774Z 13 260-279 166 163 166 306 298 306 F T F <---
179762024-02-03T17:30:28.775Z 14 280-299 155 155 155 284 284 284 F F F
179772024-02-03T17:30:28.775Z 15 300-319 151 151 151 279 279 279 F F F
179782024-02-03T17:30:28.775Z 16 320-339 159 159 159 292 292 292 F F F
179792024-02-03T17:30:28.775Z 17 340-359 160 160 160 294 294 294 T F T <---
179802024-02-03T17:30:28.775Z 18 360-379 165 165 165 303 303 303 F F F
179812024-02-03T17:30:28.775Z 19 380-399 158 158 158 291 291 291 F F F
179822024-02-03T17:30:28.775Z 20 400-419 162 162 162 297 297 297 F F F
179832024-02-03T17:30:28.775Z 21 420-439 161 161 161 296 296 296 F F F
179842024-02-03T17:30:28.775Z 22 440-459 166 154 166 306 282 306 F F F <---
179852024-02-03T17:30:28.775Z 23 460-479 166 166 166 305 305 305 F F F
179862024-02-03T17:30:28.775Z 24 480-499 164 164 164 301 301 301 F F F
179872024-02-03T17:30:28.775Z 25 500-519 164 164 164 302 302 302 T F T <---
179882024-02-03T17:30:28.775Z 26 520-539 165 165 165 303 303 303 F F F
179892024-02-03T17:30:28.775Z 27 540-559 166 165 166 306 303 306 F F F <---
179902024-02-03T17:30:28.775Z 28 560-579 166 165 166 306 303 306 T F T <---
179912024-02-03T17:30:28.775Z 29 580-599 166 160 166 306 293 306 F F F <---
179922024-02-03T17:30:28.775ZMax gen: 166, Max flush: 306
179932024-02-03T17:30:28.775ZError: Difference in extent metadata found!
179942024-02-03T17:30:28.779ZA Difference in extent metadata is expected here
179952024-02-03T17:30:28.779ZOn loop 83
179962024-02-03T17:30:28.779Z
179972024-02-03T17:30:28.779ZVerifying data now
179982024-02-03T17:30:33.950ZNo change after verify
179992024-02-03T17:30:33.950ZLoop: 83 Downstairs dump after verify (and repair):
180002024-02-03T17:30:33.978ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180012024-02-03T17:30:33.978Z 0 000-019 161 161 161 296 296 296 F F F
180022024-02-03T17:30:33.978Z 1 020-039 167 167 167 307 307 307 F F F
180032024-02-03T17:30:33.978Z 2 040-059 167 167 167 307 307 307 F F F
180042024-02-03T17:30:33.981Z 3 060-079 166 166 166 306 306 306 F F F
180052024-02-03T17:30:33.982Z 4 080-099 166 166 166 306 306 306 F F F
180062024-02-03T17:30:33.982Z 5 100-119 167 167 167 307 307 307 F F F
180072024-02-03T17:30:33.982Z 6 120-139 154 154 154 282 282 282 F F F
180082024-02-03T17:30:33.982Z 7 140-159 159 159 159 292 292 292 F F F
180092024-02-03T17:30:33.982Z 8 160-179 163 163 163 298 298 298 F F F
180102024-02-03T17:30:33.982Z 9 180-199 163 163 163 298 298 298 F F F
180112024-02-03T17:30:33.982Z 10 200-219 163 163 163 298 298 298 F F F
180122024-02-03T17:30:33.982Z 11 220-239 167 167 167 307 307 307 F F F
180132024-02-03T17:30:33.982Z 12 240-259 164 164 164 299 299 299 F F F
180142024-02-03T17:30:33.982Z 13 260-279 166 166 166 306 306 306 F F F
180152024-02-03T17:30:33.982Z 14 280-299 155 155 155 284 284 284 F F F
180162024-02-03T17:30:33.982Z 15 300-319 151 151 151 279 279 279 F F F
180172024-02-03T17:30:33.982Z 16 320-339 159 159 159 292 292 292 F F F
180182024-02-03T17:30:33.982Z 17 340-359 167 167 167 307 307 307 F F F
180192024-02-03T17:30:33.982Z 18 360-379 165 165 165 303 303 303 F F F
180202024-02-03T17:30:33.982Z 19 380-399 158 158 158 291 291 291 F F F
180212024-02-03T17:30:33.982Z 20 400-419 162 162 162 297 297 297 F F F
180222024-02-03T17:30:33.982Z 21 420-439 161 161 161 296 296 296 F F F
180232024-02-03T17:30:33.982Z 22 440-459 166 166 166 306 306 306 F F F
180242024-02-03T17:30:33.983Z 23 460-479 166 166 166 305 305 305 F F F
180252024-02-03T17:30:33.983Z 24 480-499 164 164 164 301 301 301 F F F
180262024-02-03T17:30:33.983Z 25 500-519 167 167 167 307 307 307 F F F
180272024-02-03T17:30:33.983Z 26 520-539 165 165 165 303 303 303 F F F
180282024-02-03T17:30:33.983Z 27 540-559 166 166 166 306 306 306 F F F
180292024-02-03T17:30:33.983Z 28 560-579 167 167 167 307 307 307 F F F
180302024-02-03T17:30:33.983Z 29 580-599 166 166 166 306 306 306 F F F
180312024-02-03T17:30:33.983ZMax gen: 167, Max flush: 307
180322024-02-03T17:30:33.983Z
180332024-02-03T17:30:33.983ZBegin loop 84
180342024-02-03T17:30:33.983ZDownstairs to restart: 1
180352024-02-03T17:30:34.014Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.189261795Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807}
180362024-02-03T17:30:34.014Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.189763611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807}
180372024-02-03T17:30:34.014Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.189791563Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807}
180382024-02-03T17:30:34.020Z{"msg":"Crucible 238e819a-2f4e-42a9-9a39-3dbcaa4ff7ee has session id: b0e348d5-dd96-49b5-a6f4-53d13ef604af","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.189823079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180392024-02-03T17:30:34.020Z{"msg":"Upstairs opts: Upstairs UUID: 238e819a-2f4e-42a9-9a39-3dbcaa4ff7ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.189913344Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180402024-02-03T17:30:34.020Z{Crucible runtime is spawned
180412024-02-03T17:30:34.021Z"msg":"Crucible stats registered with UUID: 238e819a-2f4e-42a9-9a39-3dbcaa4ff7ee","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.19000569Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180422024-02-03T17:30:34.021Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190059941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807}
180432024-02-03T17:30:34.021Z{"msg":"238e819a-2f4e-42a9-9a39-3dbcaa4ff7ee active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190303209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180442024-02-03T17:30:34.021Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190360086Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180452024-02-03T17:30:34.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190476002Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180462024-02-03T17:30:34.021Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190510223Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180472024-02-03T17:30:34.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190548696Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180482024-02-03T17:30:34.021Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190605717Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180492024-02-03T17:30:34.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190632106Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180502024-02-03T17:30:34.021Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190654197Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180512024-02-03T17:30:34.021Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190746145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180522024-02-03T17:30:34.021Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190788337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180532024-02-03T17:30:34.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.190845399Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180542024-02-03T17:30:34.021Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:33.191141853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180552024-02-03T17:30:34.021Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:33.191218044Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180562024-02-03T17:30:34.021Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.191310453Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180572024-02-03T17:30:34.021Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:33.191359458Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180582024-02-03T17:30:34.021Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.191398071Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180592024-02-03T17:30:34.021Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.191461796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180602024-02-03T17:30:34.021Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.19161721Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180612024-02-03T17:30:34.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.192367153Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180622024-02-03T17:30:34.022Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.192440572Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180632024-02-03T17:30:34.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.192484526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180642024-02-03T17:30:34.022Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.192577575Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180652024-02-03T17:30:34.022Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.193349182Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180662024-02-03T17:30:34.022Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.193529029Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180672024-02-03T17:30:34.026Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.194237638Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180682024-02-03T17:30:34.026Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.194279831Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180692024-02-03T17:30:34.026Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.194304276Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180702024-02-03T17:30:34.026Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.194343078Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180712024-02-03T17:30:34.026Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.194389684Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180722024-02-03T17:30:34.026Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:33.19441461Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180732024-02-03T17:30:35.016Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.192413966Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180742024-02-03T17:30:35.017Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.19251765Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180752024-02-03T17:30:35.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.193827277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180762024-02-03T17:30:35.022Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.193934998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180772024-02-03T17:30:35.022Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.194629347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180782024-02-03T17:30:35.022Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195492341Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180792024-02-03T17:30:35.022Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.19554504Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180802024-02-03T17:30:35.022Z{"msg":"[0]R flush_numbers[0..12]: [296, 307, 307, 306, 306, 307, 282, 292, 298, 298, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195571885Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180812024-02-03T17:30:35.022Z{"msg":"[0]R generation[0..12]: [161, 167, 167, 166, 166, 167, 154, 159, 163, 163, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195595115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180822024-02-03T17:30:35.022Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195642796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180832024-02-03T17:30:35.022Z{"msg":"[1]R flush_numbers[0..12]: [296, 307, 307, 306, 306, 307, 282, 292, 298, 298, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195668913Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180842024-02-03T17:30:35.022Z{"msg":"[1]R generation[0..12]: [161, 167, 167, 166, 166, 167, 154, 159, 163, 163, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195693756Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180852024-02-03T17:30:35.022Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195717363Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180862024-02-03T17:30:35.022Z{"msg":"[2]R flush_numbers[0..12]: [296, 307, 307, 306, 306, 307, 282, 292, 298, 298, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195759703Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180872024-02-03T17:30:35.022Z{"msg":"[2]R generation[0..12]: [161, 167, 167, 166, 166, 167, 154, 159, 163, 163, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195784204Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180882024-02-03T17:30:35.022Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195807828Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180892024-02-03T17:30:35.022Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195848469Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180902024-02-03T17:30:35.022Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195872895Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180912024-02-03T17:30:35.022Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195894337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180922024-02-03T17:30:35.022Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195948801Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180932024-02-03T17:30:35.022Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195973042Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180942024-02-03T17:30:35.023Z{"msg":"Transition from WaitQuorum to Active","v":0Wait for a query_work_queue command to finish before sending IO
180952024-02-03T17:30:35.023Z,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.195998345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180962024-02-03T17:30:35.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.196022164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180972024-02-03T17:30:35.023Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.196066094Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180982024-02-03T17:30:35.023Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.19609077Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
180992024-02-03T17:30:35.023Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.196111667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
181002024-02-03T17:30:35.023Z{"msg":"238e819a-2f4e-42a9-9a39-3dbcaa4ff7ee is now active with session: b0e348d5-dd96-49b5-a6f4-53d13ef604af","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.196132724Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
181012024-02-03T17:30:35.023Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.196154168Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807,"session_id":"b0e348d5-dd96-49b5-a6f4-53d13ef604af"}
181022024-02-03T17:30:35.023Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:34.196174526Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807}
181032024-02-03T17:30:35.029ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181042024-02-03T17:30:35.029ZLoading write count information from file "/tmp/test_repair_verify.data"
181052024-02-03T17:30:35.029ZRun Repair workload
181062024-02-03T17:30:35.029Z01/30 Write block 358 len 4096 data: 17
181072024-02-03T17:30:35.029Z02/30 Read block 494 len 4096
181082024-02-03T17:30:35.029Z03/30 Write block 417 len 4096 data: 9
181092024-02-03T17:30:35.029Z04/30 Read block 500 len 8192
181102024-02-03T17:30:35.033Z05/30 Flush
181112024-02-03T17:30:35.033Z06/30 Write block 167 len 32768 data: 9 8 12 12 12 9 9 11
181122024-02-03T17:30:35.033Z07/30 Read block 525 len 40960
181132024-02-03T17:30:35.036Z08/30 Read block 14 len 20480
181142024-02-03T17:30:35.036Z09/30 Flush
181152024-02-03T17:30:35.039Z10/30 Flush
181162024-02-03T17:30:35.039Z11/30 Write block 588 len 12288 data: 15 17 15
181172024-02-03T17:30:35.039Z12/30 Write block 107 len 16384 data: 18 17 17 17
181182024-02-03T17:30:35.042Z13/30 Read block 532 len 16384
181192024-02-03T17:30:35.043Z14/30 Read block 473 len 40960
181202024-02-03T17:30:35.045Z15/30 Read block 23 len 40960
181212024-02-03T17:30:35.045Z16/30 Read block 60 len 32768
181222024-02-03T17:30:35.048Z17/30 Write block 574 len 40960 data: 9 11 13 15 19 19 19 19 19 17
181232024-02-03T17:30:35.048Z18/30 Read block 376 len 4096
181242024-02-03T17:30:35.092Z19/30 Write block 294 len 32768 data: 12 10 12 13 9 6 9 8
181252024-02-03T17:30:35.092Z20/30 Write block 306 len 8192 data: 15 12
181262024-02-03T17:30:35.092Z21/30 Read block 361 len 4096
181272024-02-03T17:30:35.092Z22/30 Read block 479 len 40960
181282024-02-03T17:30:35.092Z23/30 Read block 500 len 36864
181292024-02-03T17:30:35.093Z24/30 Read block 212 len 20480
181302024-02-03T17:30:35.093Z25/30 Read block 499 len 8192
181312024-02-03T17:30:35.093Z26/30 Read block 449 len 32768
181322024-02-03T17:30:35.093Z27/30 Write block 583 len 40960 data: 18 18 16 19 15 16 18 16 16 14
181332024-02-03T17:30:35.093Z28/30 Read block 82 len 20480
181342024-02-03T17:30:35.093Z29/30 Read block 392 len 36864
181352024-02-03T17:30:35.093Z30/30 Read block 535 len 40960
181362024-02-03T17:30:35.093Z----------------------------------------------------------------
181372024-02-03T17:30:35.093Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
181382024-02-03T17:30:35.093ZStates: Active Active Active
181392024-02-03T17:30:35.093ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181402024-02-03T17:30:35.093Z 1 Acked 1000 Write 1 Done Sent Done false
181412024-02-03T17:30:35.093Z 2 Acked 1001 Read 1 Done Sent Done false
181422024-02-03T17:30:35.094Z 3 Acked 1002 Write 1 Done Sent Done false
181432024-02-03T17:30:35.094Z 4 Acked 1003 Read 2 Done Sent Done false
181442024-02-03T17:30:35.094Z 5 Acked 1004 Flush 0 Done Sent Done false
181452024-02-03T17:30:35.094Z 6 Acked 1005 Write 8 Done Sent Done false
181462024-02-03T17:30:35.094Z 7 Acked 1006 Read 10 Done Sent Done false
181472024-02-03T17:30:35.094Z 8 Acked 1007 Read 5 Done Sent Done false
181482024-02-03T17:30:35.094Z 9 Acked 1008 Flush 0 Done Sent Done false
181492024-02-03T17:30:35.094Z 10 Acked 1009 Flush 0 Done Sent Done false
181502024-02-03T17:30:35.094Z 11 Acked 1010 Write 3 Done Sent Done false
181512024-02-03T17:30:35.094Z 12 Acked 1011 Write 4 Done Sent Done false
181522024-02-03T17:30:35.094Z 13 Acked 1012 Read 4 Done Sent Done false
181532024-02-03T17:30:35.094Z 14 Acked 1013 Read 10 Done Sent Done false
181542024-02-03T17:30:35.094Z 15 Acked 1014 Read 10 Done Sent Done false
181552024-02-03T17:30:35.094Z 16 Acked 1015 Read 8 Done Sent Done false
181562024-02-03T17:30:35.095Z 17 Acked 1016 Write 10 Done Sent Done false
181572024-02-03T17:30:35.095Z 18 Acked 1017 Read 1 Done Sent Done false
181582024-02-03T17:30:35.095Z 19 Acked 1018 Write 8 Done Sent Done false
181592024-02-03T17:30:35.095Z 20 Acked 1019 Write 2 Done Sent Done false
181602024-02-03T17:30:35.095Z 21 Acked 1020 Read 1 Done Sent Done false
181612024-02-03T17:30:35.095Z 22 Acked 1021 Read 10 Done Sent Done false
181622024-02-03T17:30:35.095Z 23 Acked 1022 Read 9 Done Sent Done false
181632024-02-03T17:30:35.095Z 24 Acked 1023 Read 5 Done Sent Done false
181642024-02-03T17:30:35.095Z 25 Acked 1024 Read 2 Done Sent Done false
181652024-02-03T17:30:35.095Z 26 Acked 1025 Read 8 Done Sent Done false
181662024-02-03T17:30:35.095Z 27 Acked 1026 Write 10 Done Sent Done false
181672024-02-03T17:30:35.095Z 28 Acked 1027 Read 5 Done Sent Done false
181682024-02-03T17:30:35.095Z 29 Acked 1028 Read 9 Done Sent Done false
181692024-02-03T17:30:35.096Z 30 Acked 1029 Read 10 Done Sent Done false
181702024-02-03T17:30:35.096Z STATES DS:0 DS:1 DS:2 TOTAL
181712024-02-03T17:30:35.096Z New 0 0 0 0
181722024-02-03T17:30:35.096Z Sent 0 30 0 30
181732024-02-03T17:30:35.096Z Done 30 0 30 60
181742024-02-03T17:30:35.096Z Skipped 0 0 0 0
181752024-02-03T17:30:35.096Z Error 0 0 0 0
181762024-02-03T17:30:35.096ZLast Flush: 1009 0 1009
181772024-02-03T17:30:35.096ZDownstairs last five completed:
181782024-02-03T17:30:35.096ZUpstairs last five completed: 30 29 28 27 26
181792024-02-03T17:30:35.096Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:34.244169862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1807}
181802024-02-03T17:30:35.096ZWrote out file "/tmp/test_repair_verify.data"
181812024-02-03T17:30:35.096Z
181822024-02-03T17:30:35.096ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181832024-02-03T17:30:35.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181842024-02-03T17:30:35.110Z 0 000-019 161 161 161 296 296 296 F F F
181852024-02-03T17:30:35.110Z 1 020-039 167 167 167 307 307 307 F F F
181862024-02-03T17:30:35.110Z 2 040-059 167 167 167 307 307 307 F F F
181872024-02-03T17:30:35.115Z 3 060-079 166 166 166 306 306 306 F F F
181882024-02-03T17:30:35.115Z 4 080-099 166 166 166 306 306 306 F F F
181892024-02-03T17:30:35.115Z 5 100-119 167 167 167 307 307 307 T F T <---
181902024-02-03T17:30:35.115Z 6 120-139 154 154 154 282 282 282 F F F
181912024-02-03T17:30:35.115Z 7 140-159 159 159 159 292 292 292 F F F
181922024-02-03T17:30:35.115Z 8 160-179 168 163 168 309 298 309 F F F <---
181932024-02-03T17:30:35.115Z 9 180-199 163 163 163 298 298 298 F F F
181942024-02-03T17:30:35.115Z 10 200-219 163 163 163 298 298 298 F F F
181952024-02-03T17:30:35.115Z 11 220-239 167 167 167 307 307 307 F F F
181962024-02-03T17:30:35.115Z 12 240-259 164 164 164 299 299 299 F F F
181972024-02-03T17:30:35.115Z 13 260-279 166 166 166 306 306 306 F F F
181982024-02-03T17:30:35.115Z 14 280-299 155 155 155 284 284 284 T F T <---
181992024-02-03T17:30:35.115Z 15 300-319 151 151 151 279 279 279 T F T <---
182002024-02-03T17:30:35.115Z 16 320-339 159 159 159 292 292 292 F F F
182012024-02-03T17:30:35.115Z 17 340-359 168 167 168 308 307 308 F F F <---
182022024-02-03T17:30:35.115Z 18 360-379 165 165 165 303 303 303 F F F
182032024-02-03T17:30:35.115Z 19 380-399 158 158 158 291 291 291 F F F
182042024-02-03T17:30:35.115Z 20 400-419 168 162 168 308 297 308 F F F <---
182052024-02-03T17:30:35.115Z 21 420-439 161 161 161 296 296 296 F F F
182062024-02-03T17:30:35.115Z 22 440-459 166 166 166 306 306 306 F F F
182072024-02-03T17:30:35.116Z 23 460-479 166 166 166 305 305 305 F F F
182082024-02-03T17:30:35.116Z 24 480-499 164 164 164 301 301 301 F F F
182092024-02-03T17:30:35.116Z 25 500-519 167 167 167 307 307 307 F F F
182102024-02-03T17:30:35.116Z 26 520-539 165 165 165 303 303 303 F F F
182112024-02-03T17:30:35.116Z 27 540-559 166 166 166 306 306 306 F F F
182122024-02-03T17:30:35.116Z 28 560-579 167 167 167 307 307 307 T F T <---
182132024-02-03T17:30:35.116Z 29 580-599 166 166 166 306 306 306 T F T <---
182142024-02-03T17:30:35.116ZMax gen: 168, Max flush: 309
182152024-02-03T17:30:35.116ZError: Difference in extent metadata found!
182162024-02-03T17:30:35.116ZA Difference in extent metadata is expected here
182172024-02-03T17:30:35.116ZOn loop 84
182182024-02-03T17:30:35.116Z
182192024-02-03T17:30:35.120ZVerifying data now
182202024-02-03T17:30:39.893ZNo change after verify
182212024-02-03T17:30:39.894ZLoop: 84 Downstairs dump after verify (and repair):
182222024-02-03T17:30:39.919ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182232024-02-03T17:30:39.919Z 0 000-019 161 161 161 296 296 296 F F F
182242024-02-03T17:30:39.919Z 1 020-039 167 167 167 307 307 307 F F F
182252024-02-03T17:30:39.919Z 2 040-059 167 167 167 307 307 307 F F F
182262024-02-03T17:30:39.919Z 3 060-079 166 166 166 306 306 306 F F F
182272024-02-03T17:30:39.919Z 4 080-099 166 166 166 306 306 306 F F F
182282024-02-03T17:30:39.924Z 5 100-119 169 169 169 310 310 310 F F F
182292024-02-03T17:30:39.925Z 6 120-139 154 154 154 282 282 282 F F F
182302024-02-03T17:30:39.925Z 7 140-159 159 159 159 292 292 292 F F F
182312024-02-03T17:30:39.925Z 8 160-179 168 168 168 309 309 309 F F F
182322024-02-03T17:30:39.925Z 9 180-199 163 163 163 298 298 298 F F F
182332024-02-03T17:30:39.925Z 10 200-219 163 163 163 298 298 298 F F F
182342024-02-03T17:30:39.925Z 11 220-239 167 167 167 307 307 307 F F F
182352024-02-03T17:30:39.925Z 12 240-259 164 164 164 299 299 299 F F F
182362024-02-03T17:30:39.925Z 13 260-279 166 166 166 306 306 306 F F F
182372024-02-03T17:30:39.925Z 14 280-299 169 169 169 310 310 310 F F F
182382024-02-03T17:30:39.925Z 15 300-319 169 169 169 310 310 310 F F F
182392024-02-03T17:30:39.925Z 16 320-339 159 159 159 292 292 292 F F F
182402024-02-03T17:30:39.925Z 17 340-359 168 168 168 308 308 308 F F F
182412024-02-03T17:30:39.925Z 18 360-379 165 165 165 303 303 303 F F F
182422024-02-03T17:30:39.925Z 19 380-399 158 158 158 291 291 291 F F F
182432024-02-03T17:30:39.925Z 20 400-419 168 168 168 308 308 308 F F F
182442024-02-03T17:30:39.925Z 21 420-439 161 161 161 296 296 296 F F F
182452024-02-03T17:30:39.925Z 22 440-459 166 166 166 306 306 306 F F F
182462024-02-03T17:30:39.925Z 23 460-479 166 166 166 305 305 305 F F F
182472024-02-03T17:30:39.925Z 24 480-499 164 164 164 301 301 301 F F F
182482024-02-03T17:30:39.925Z 25 500-519 167 167 167 307 307 307 F F F
182492024-02-03T17:30:39.925Z 26 520-539 165 165 165 303 303 303 F F F
182502024-02-03T17:30:39.926Z 27 540-559 166 166 166 306 306 306 F F F
182512024-02-03T17:30:39.926Z 28 560-579 169 169 169 310 310 310 F F F
182522024-02-03T17:30:39.926Z 29 580-599 169 169 169 310 310 310 F F F
182532024-02-03T17:30:39.926ZMax gen: 169, Max flush: 310
182542024-02-03T17:30:39.930Z
182552024-02-03T17:30:39.930ZBegin loop 85
182562024-02-03T17:30:39.930ZDownstairs to restart: 1
182572024-02-03T17:30:39.957Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.132856699Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815}
182582024-02-03T17:30:39.957Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.13333243Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815}
182592024-02-03T17:30:39.961Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.133408456Z"Crucible runtime is spawned
182602024-02-03T17:30:39.961Z,"hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815}
182612024-02-03T17:30:39.961Z{"msg":"Crucible 71106a2b-4a6d-4130-b841-2db80e0df3ba has session id: 7da07258-913f-419a-9891-a79f0e175853","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.133686223Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182622024-02-03T17:30:39.961Z{"msg":"Upstairs opts: Upstairs UUID: 71106a2b-4a6d-4130-b841-2db80e0df3ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.133790457Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182632024-02-03T17:30:39.961Z{"msg":"Crucible stats registered with UUID: 71106a2b-4a6d-4130-b841-2db80e0df3ba","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.133863619Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182642024-02-03T17:30:39.961Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.133931685Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815}
182652024-02-03T17:30:39.961Z{"msg":"71106a2b-4a6d-4130-b841-2db80e0df3ba active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.133966686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182662024-02-03T17:30:39.961Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134002491Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182672024-02-03T17:30:39.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134041794Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182682024-02-03T17:30:39.961Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.13412792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182692024-02-03T17:30:39.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134196967Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182702024-02-03T17:30:39.961Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134235973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182712024-02-03T17:30:39.961Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134275227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182722024-02-03T17:30:39.961Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134313005Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182732024-02-03T17:30:39.962Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134383745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182742024-02-03T17:30:39.962Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134456973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182752024-02-03T17:30:39.962Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134500347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182762024-02-03T17:30:39.962Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134566728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182772024-02-03T17:30:39.962Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.134635667Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182782024-02-03T17:30:39.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.135852472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182792024-02-03T17:30:39.962Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.135914276Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182802024-02-03T17:30:39.962Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.136009259Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182812024-02-03T17:30:39.962Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.136058942Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182822024-02-03T17:30:39.966Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.136944284Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182832024-02-03T17:30:39.966Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.137269522Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182842024-02-03T17:30:39.966Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.137363441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182852024-02-03T17:30:39.967Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.137416235Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182862024-02-03T17:30:39.967Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.13795122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182872024-02-03T17:30:39.967Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.138083223Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182882024-02-03T17:30:39.967Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.138173432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182892024-02-03T17:30:39.967Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.138218299Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182902024-02-03T17:30:39.967Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.138283127Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182912024-02-03T17:30:39.967Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.138328596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182922024-02-03T17:30:39.967Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.13838464Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182932024-02-03T17:30:39.967Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.138989145Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182942024-02-03T17:30:39.967Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139127227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182952024-02-03T17:30:39.967Z{"msg":"[0]R flush_numbers[0..12]: [296, 307, 307, 306, 306, 310, 282, 292, 309, 298, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139157268Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182962024-02-03T17:30:39.967Z{"msg":"[0]R generation[0..12]: [161, 167, 167, 166, 166, 169, 154, 159, 168, 163, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139201723Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182972024-02-03T17:30:39.967Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139228324Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182982024-02-03T17:30:39.968Z{"msg":"[1]R flush_numbers[0..12]: [296, 307, 307, 306, 306, 310, 282, 292, 309, 298, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139281315Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
182992024-02-03T17:30:39.968Z{"msg":"[1]R generation[0..12]: [161, 167, 167, 166, 166, 169, 154, 159, 168, 163, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139306968Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183002024-02-03T17:30:39.968ZWait for a query_work_queue command to finish before sending IO
183012024-02-03T17:30:39.968Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139339404Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183022024-02-03T17:30:39.968Z{"msg":"[2]R flush_numbers[0..12]: [296, 307, 307, 306, 306, 310, 282, 292, 309, 298, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139363399Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183032024-02-03T17:30:39.968Z{"msg":"[2]R generation[0..12]: [161, 167, 167, 166, 166, 169, 154, 159, 168, 163, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139405517Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183042024-02-03T17:30:39.968Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139457164Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183052024-02-03T17:30:39.968Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139486639Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183062024-02-03T17:30:39.968Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139533704Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183072024-02-03T17:30:39.968Z{"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139557387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183082024-02-03T17:30:39.968Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139579307Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183092024-02-03T17:30:39.968Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139601328Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183102024-02-03T17:30:39.968Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139664864Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183112024-02-03T17:30:39.969Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139731225Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183122024-02-03T17:30:39.969Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.13975745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183132024-02-03T17:30:39.969Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139814764Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183142024-02-03T17:30:39.969Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139848555Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183152024-02-03T17:30:39.969Z{"msg":"71106a2b-4a6d-4130-b841-2db80e0df3ba is now active with session: 7da07258-913f-419a-9891-a79f0e175853","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139907627Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183162024-02-03T17:30:39.969Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.139970453Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815,"session_id":"7da07258-913f-419a-9891-a79f0e175853"}
183172024-02-03T17:30:39.969ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
183182024-02-03T17:30:39.969Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:39.140040482Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815}
183192024-02-03T17:30:39.969ZLoading write count information from file "/tmp/test_repair_verify.data"
183202024-02-03T17:30:39.969ZRun Repair workload
183212024-02-03T17:30:39.969Z01/30 Write block 381 len 12288 data: 12 11 10
183222024-02-03T17:30:39.973Z02/30 Read block 246 len 24576
183232024-02-03T17:30:39.973Z03/30 Write block 19 len 36864 data: 8 5 8 10 10 14 13 11 11
183242024-02-03T17:30:39.973Z04/30 Read block 347 len 20480
183252024-02-03T17:30:39.973Z05/30 Read block 407 len 8192
183262024-02-03T17:30:39.977Z06/30 Read block 426 len 20480
183272024-02-03T17:30:39.977Z07/30 Read block 69 len 20480
183282024-02-03T17:30:39.977Z08/30 Read block 97 len 36864
183292024-02-03T17:30:39.980Z09/30 Write block 249 len 36864 data: 14 19 12 9 11 13 13 13 11
183302024-02-03T17:30:39.980Z10/30 Read block 74 len 32768
183312024-02-03T17:30:39.984Z11/30 Read block 445 len 28672
183322024-02-03T17:30:39.984Z12/30 Read block 412 len 28672
183332024-02-03T17:30:39.987Z13/30 Write block 293 len 28672 data: 12 13 11 13 14 10 7
183342024-02-03T17:30:39.987Z14/30 Read block 321 len 8192
183352024-02-03T17:30:39.991Z15/30 Write block 181 len 4096 data: 14
183362024-02-03T17:30:39.991Z16/30 Read block 282 len 36864
183372024-02-03T17:30:39.991Z17/30 Read block 239 len 16384
183382024-02-03T17:30:39.994Z18/30 Read block 353 len 40960
183392024-02-03T17:30:39.994Z19/30 Read block 393 len 36864
183402024-02-03T17:30:39.999Z20/30 Read block 206 len 40960
183412024-02-03T17:30:39.999Z21/30 Write block 308 len 16384 data: 14 14 12 11
183422024-02-03T17:30:40.003Z22/30 Write block 409 len 40960 data: 13 11 11 10 13 10 6 7 10 13
183432024-02-03T17:30:40.003Z23/30 Read block 576 len 32768
183442024-02-03T17:30:40.003Z24/30 Read block 206 len 28672
183452024-02-03T17:30:40.006Z25/30 Read block 284 len 32768
183462024-02-03T17:30:40.007Z26/30 Write block 581 len 36864 data: 20 20 19 19 17 20 16 17 19
183472024-02-03T17:30:40.009Z27/30 Write block 430 len 36864 data: 10 13 13 11 14 14 13 14 15
183482024-02-03T17:30:40.010Z28/30 Write block 150 len 36864 data: 12 13 9 10 11 14 14 14 14
183492024-02-03T17:30:40.013Z29/30 Read block 175 len 20480
183502024-02-03T17:30:40.013Z30/30 Write block 490 len 12288 data: 14 13 12
183512024-02-03T17:30:40.013Z----------------------------------------------------------------
183522024-02-03T17:30:40.013Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183532024-02-03T17:30:40.013ZStates: Active Active Active
183542024-02-03T17:30:40.014ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183552024-02-03T17:30:40.014Z 1 Acked 1000 Write 3 Done Done Done false
183562024-02-03T17:30:40.014Z 2 Acked 1001 Read 6 Done Done Done false
183572024-02-03T17:30:40.014Z 3 Acked 1002 Write 9 Done Done Done false
183582024-02-03T17:30:40.014Z 4 Acked 1003 Read 5 Done Done Done false
183592024-02-03T17:30:40.014Z 5 Acked 1004 Read 2 Done Sent Done false
183602024-02-03T17:30:40.014Z 6 Acked 1005 Read 5 Done Sent Done false
183612024-02-03T17:30:40.018Z 7 Acked 1006 Read 5 Done Sent Done false
183622024-02-03T17:30:40.018Z 8 Acked 1007 Read 9 Done Sent Done false
183632024-02-03T17:30:40.018Z 9 Acked 1008 Write 9 Done Sent Done false
183642024-02-03T17:30:40.018Z 10 Acked 1009 Read 8 Done Sent Done false
183652024-02-03T17:30:40.018Z 11 Acked 1010 Read 7 Done Sent Done false
183662024-02-03T17:30:40.018Z 12 Acked 1011 Read 7 Done Sent Done false
183672024-02-03T17:30:40.019Z 13 Acked 1012 Write 7 Done Sent Done false
183682024-02-03T17:30:40.019Z 14 Acked 1013 Read 2 Done Sent Done false
183692024-02-03T17:30:40.019Z 15 Acked 1014 Write 1 Done Sent Done false
183702024-02-03T17:30:40.019Z 16 Acked 1015 Read 9 Done Sent Done false
183712024-02-03T17:30:40.019Z 17 Acked 1016 Read 4 Done Sent Done false
183722024-02-03T17:30:40.019Z 18 Acked 1017 Read 10 Done Sent Done false
183732024-02-03T17:30:40.019Z 19 Acked 1018 Read 9 Done Sent Done false
183742024-02-03T17:30:40.019Z 20 Acked 1019 Read 10 Done Sent Done false
183752024-02-03T17:30:40.019Z 21 Acked 1020 Write 4 Done Sent Done false
183762024-02-03T17:30:40.019Z 22 Acked 1021 Write 10 Done Sent Done false
183772024-02-03T17:30:40.019Z 23 Acked 1022 Read 8 Done Sent Done false
183782024-02-03T17:30:40.019Z 24 Acked 1023 Read 7 Done Sent Done false
183792024-02-03T17:30:40.019Z 25 Acked 1024 Read 8 Done Sent Done false
183802024-02-03T17:30:40.020Z 26 Acked 1025 Write 9 Done Sent Done false
183812024-02-03T17:30:40.020Z 27 Acked 1026 Write 9 Done Sent Done false
183822024-02-03T17:30:40.020Z 28 Acked 1027 Write 9 Done Sent Done false
183832024-02-03T17:30:40.020Z 29 Acked 1028 Read 5 Done Sent Done false
183842024-02-03T17:30:40.020Z 30 Acked 1029 Write 3 Sent Sent Sent false
183852024-02-03T17:30:40.020Z STATES DS:0 DS:1 DS:2 TOTAL
183862024-02-03T17:30:40.020Z New 0 0 0 0
183872024-02-03T17:30:40.020Z Sent 1 26 1 28
183882024-02-03T17:30:40.020Z Done 29 4 29 62
183892024-02-03T17:30:40.020Z Skipped 0 0 0 0
183902024-02-03T17:30:40.020Z Error 0 0 0 0
183912024-02-03T17:30:40.020ZLast Flush: 0 0 0
183922024-02-03T17:30:40.020ZDownstairs last five completed:
183932024-02-03T17:30:40.021ZUpstairs last five completed: 30 29 28 27 26
183942024-02-03T17:30:40.021Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:39.190031426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1815}
183952024-02-03T17:30:40.021ZWrote out file "/tmp/test_repair_verify.data"
183962024-02-03T17:30:40.024Z
183972024-02-03T17:30:40.029ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183982024-02-03T17:30:40.072ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183992024-02-03T17:30:40.072Z 0 000-019 161 161 161 296 296 296 T T T
184002024-02-03T17:30:40.076Z 1 020-039 167 167 167 307 307 307 T T T
184012024-02-03T17:30:40.077Z 2 040-059 167 167 167 307 307 307 F F F
184022024-02-03T17:30:40.077Z 3 060-079 166 166 166 306 306 306 F F F
184032024-02-03T17:30:40.077Z 4 080-099 166 166 166 306 306 306 F F F
184042024-02-03T17:30:40.077Z 5 100-119 169 169 169 310 310 310 F F F
184052024-02-03T17:30:40.077Z 6 120-139 154 154 154 282 282 282 F F F
184062024-02-03T17:30:40.077Z 7 140-159 159 159 159 292 292 292 T F T <---
184072024-02-03T17:30:40.077Z 8 160-179 168 168 168 309 309 309 F F F
184082024-02-03T17:30:40.077Z 9 180-199 163 163 163 298 298 298 T F T <---
184092024-02-03T17:30:40.077Z 10 200-219 163 163 163 298 298 298 F F F
184102024-02-03T17:30:40.077Z 11 220-239 167 167 167 307 307 307 F F F
184112024-02-03T17:30:40.077Z 12 240-259 164 164 164 299 299 299 T F T <---
184122024-02-03T17:30:40.077Z 13 260-279 166 166 166 306 306 306 F F F
184132024-02-03T17:30:40.077Z 14 280-299 169 169 169 310 310 310 T F T <---
184142024-02-03T17:30:40.078Z 15 300-319 169 169 169 310 310 310 T F T <---
184152024-02-03T17:30:40.078Z 16 320-339 159 159 159 292 292 292 F F F
184162024-02-03T17:30:40.078Z 17 340-359 168 168 168 308 308 308 F F F
184172024-02-03T17:30:40.078Z 18 360-379 165 165 165 303 303 303 F F F
184182024-02-03T17:30:40.078Z 19 380-399 158 158 158 291 291 291 T T T
184192024-02-03T17:30:40.078Z 20 400-419 168 168 168 308 308 308 T F T <---
184202024-02-03T17:30:40.078Z 21 420-439 161 161 161 296 296 296 T F T <---
184212024-02-03T17:30:40.078Z 22 440-459 166 166 166 306 306 306 F F F
184222024-02-03T17:30:40.078Z 23 460-479 166 166 166 305 305 305 F F F
184232024-02-03T17:30:40.078Z 24 480-499 164 164 164 301 301 301 T F T <---
184242024-02-03T17:30:40.078Z 25 500-519 167 167 167 307 307 307 F F F
184252024-02-03T17:30:40.078Z 26 520-539 165 165 165 303 303 303 F F F
184262024-02-03T17:30:40.078Z 27 540-559 166 166 166 306 306 306 F F F
184272024-02-03T17:30:40.078Z 28 560-579 169 169 169 310 310 310 F F F
184282024-02-03T17:30:40.079Z 29 580-599 169 169 169 310 310 310 T F T <---
184292024-02-03T17:30:40.079ZMax gen: 169, Max flush: 310
184302024-02-03T17:30:40.079ZError: Difference in extent metadata found!
184312024-02-03T17:30:40.079ZA Difference in extent metadata is expected here
184322024-02-03T17:30:40.079ZOn loop 85
184332024-02-03T17:30:40.079Z
184342024-02-03T17:30:40.079ZVerifying data now
184352024-02-03T17:30:46.171ZNo change after verify
184362024-02-03T17:30:46.171ZLoop: 85 Downstairs dump after verify (and repair):
184372024-02-03T17:30:46.202ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184382024-02-03T17:30:46.202Z 0 000-019 170 170 170 311 311 311 F F F
184392024-02-03T17:30:46.206Z 1 020-039 170 170 170 311 311 311 F F F
184402024-02-03T17:30:46.206Z 2 040-059 167 167 167 307 307 307 F F F
184412024-02-03T17:30:46.206Z 3 060-079 166 166 166 306 306 306 F F F
184422024-02-03T17:30:46.206Z 4 080-099 166 166 166 306 306 306 F F F
184432024-02-03T17:30:46.207Z 5 100-119 169 169 169 310 310 310 F F F
184442024-02-03T17:30:46.207Z 6 120-139 154 154 154 282 282 282 F F F
184452024-02-03T17:30:46.207Z 7 140-159 170 170 170 311 311 311 F F F
184462024-02-03T17:30:46.207Z 8 160-179 168 168 168 309 309 309 F F F
184472024-02-03T17:30:46.207Z 9 180-199 170 170 170 311 311 311 F F F
184482024-02-03T17:30:46.207Z 10 200-219 163 163 163 298 298 298 F F F
184492024-02-03T17:30:46.207Z 11 220-239 167 167 167 307 307 307 F F F
184502024-02-03T17:30:46.207Z 12 240-259 170 170 170 311 311 311 F F F
184512024-02-03T17:30:46.207Z 13 260-279 166 166 166 306 306 306 F F F
184522024-02-03T17:30:46.207Z 14 280-299 170 170 170 311 311 311 F F F
184532024-02-03T17:30:46.207Z 15 300-319 170 170 170 311 311 311 F F F
184542024-02-03T17:30:46.207Z 16 320-339 159 159 159 292 292 292 F F F
184552024-02-03T17:30:46.207Z 17 340-359 168 168 168 308 308 308 F F F
184562024-02-03T17:30:46.207Z 18 360-379 165 165 165 303 303 303 F F F
184572024-02-03T17:30:46.207Z 19 380-399 170 170 170 311 311 311 F F F
184582024-02-03T17:30:46.208Z 20 400-419 170 170 170 311 311 311 F F F
184592024-02-03T17:30:46.208Z 21 420-439 170 170 170 311 311 311 F F F
184602024-02-03T17:30:46.208Z 22 440-459 166 166 166 306 306 306 F F F
184612024-02-03T17:30:46.208Z 23 460-479 166 166 166 305 305 305 F F F
184622024-02-03T17:30:46.208Z 24 480-499 170 170 170 311 311 311 F F F
184632024-02-03T17:30:46.208Z 25 500-519 167 167 167 307 307 307 F F F
184642024-02-03T17:30:46.208Z 26 520-539 165 165 165 303 303 303 F F F
184652024-02-03T17:30:46.208Z 27 540-559 166 166 166 306 306 306 F F F
184662024-02-03T17:30:46.208Z 28 560-579 169 169 169 310 310 310 F F F
184672024-02-03T17:30:46.208Z 29 580-599 170 170 170 311 311 311 F F F
184682024-02-03T17:30:46.208ZMax gen: 170, Max flush: 311
184692024-02-03T17:30:46.208Z
184702024-02-03T17:30:46.208ZBegin loop 86
184712024-02-03T17:30:46.208ZDownstairs to restart: 0
184722024-02-03T17:30:46.234Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.410363684Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823}
184732024-02-03T17:30:46.235Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.410917325Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823}
184742024-02-03T17:30:46.239Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.410991034Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823}
184752024-02-03T17:30:46.239Z{"msg":"Crucible c620f8b8-8cf9-4fd1-9c7a-3fd6d61857d0 has session id: fc949b17-aec5-4c5a-b268-2ab041e3f8b1","v":0,"name":"crucible","level":30Crucible runtime is spawned
184762024-02-03T17:30:46.239Z,"time":"2024-02-03T17:30:45.411033269Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184772024-02-03T17:30:46.240Z{"msg":"Upstairs opts: Upstairs UUID: c620f8b8-8cf9-4fd1-9c7a-3fd6d61857d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411196213Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184782024-02-03T17:30:46.240Z{"msg":"Crucible stats registered with UUID: c620f8b8-8cf9-4fd1-9c7a-3fd6d61857d0","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411262807Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184792024-02-03T17:30:46.240Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411289784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823}
184802024-02-03T17:30:46.240Z{"msg":"c620f8b8-8cf9-4fd1-9c7a-3fd6d61857d0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411330062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184812024-02-03T17:30:46.240Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411373295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184822024-02-03T17:30:46.240Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411398766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184832024-02-03T17:30:46.240Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411440416Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184842024-02-03T17:30:46.240Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411465104Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184852024-02-03T17:30:46.240Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.41148708Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184862024-02-03T17:30:46.240Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411510076Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184872024-02-03T17:30:46.240Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411545428Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184882024-02-03T17:30:46.240Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411649395Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184892024-02-03T17:30:46.241Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411750981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184902024-02-03T17:30:46.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.411884423Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184912024-02-03T17:30:46.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.412141831Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184922024-02-03T17:30:46.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.412173079Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184932024-02-03T17:30:46.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.413286955Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184942024-02-03T17:30:46.241Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.413397952Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184952024-02-03T17:30:46.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.413457398Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184962024-02-03T17:30:46.241Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.413502898Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184972024-02-03T17:30:46.241Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.41433653Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184982024-02-03T17:30:46.241Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.414606945Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
184992024-02-03T17:30:46.241Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.414953463Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185002024-02-03T17:30:46.241Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.415038148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185012024-02-03T17:30:46.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.415268836Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185022024-02-03T17:30:46.246Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.415329723Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185032024-02-03T17:30:46.246Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.41545847Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185042024-02-03T17:30:46.246Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.415483715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185052024-02-03T17:30:46.247Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.415506611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185062024-02-03T17:30:46.247Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.415528833Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185072024-02-03T17:30:46.247Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.415818012Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185082024-02-03T17:30:46.247Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416575552Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185092024-02-03T17:30:46.247Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.4166307Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185102024-02-03T17:30:46.247Z{"msg":"[0]R flush_numbers[0..12]: [311, 311, 307, 306, 306, 310, 282, 311, 309, 311, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416657945Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185112024-02-03T17:30:46.247Z{"msg":"[0]R generation[0..12]: [170, 170, 167, 166, 166, 169, 154, 170, 168, 170, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416680955Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185122024-02-03T17:30:46.247Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416724302Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185132024-02-03T17:30:46.247Z{"msg":"[1]R flush_numbers[0..12]: [311, 311, 307, 306, 306, 310, 282, 311, 309, 311, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416750134Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185142024-02-03T17:30:46.247Z{"msg":"[1]R generation[0..12]: [170, 170, 167, 166, 166, 169, 154, 170, 168, 170, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416773096Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185152024-02-03T17:30:46.247Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416815319Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185162024-02-03T17:30:46.247Z{"msg":"[2]R flush_numbers[0..12]: [311, 311, 307, 306, 306, 310, 282, 311, 309, 311, 298, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416841024Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185172024-02-03T17:30:46.248Z{"msg":"[2]R generation[0..12]: [170, 170, 167, 166, 166, 169, 154, 170, 168, 170, 163, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416863707Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185182024-02-03T17:30:46.248Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
185192024-02-03T17:30:46.248Z,"level":30,"time":"2024-02-03T17:30:45.416909426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185202024-02-03T17:30:46.248Z{"msg":"Max found gen is 171","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416933209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185212024-02-03T17:30:46.248Z{"msg":"Generation requested: 172 >= found:171","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.416955261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185222024-02-03T17:30:46.248Z{"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417009133Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185232024-02-03T17:30:46.248Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417032456Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185242024-02-03T17:30:46.248Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417062038Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185252024-02-03T17:30:46.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417084601Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185262024-02-03T17:30:46.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417126893Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185272024-02-03T17:30:46.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417150583Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185282024-02-03T17:30:46.248Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417172094Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185292024-02-03T17:30:46.248Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417221657Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185302024-02-03T17:30:46.249Z{"msg":"c620f8b8-8cf9-4fd1-9c7a-3fd6d61857d0 is now active with session: fc949b17-aec5-4c5a-b268-2ab041e3f8b1","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417245212Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185312024-02-03T17:30:46.249Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417265487Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823,"session_id":"fc949b17-aec5-4c5a-b268-2ab041e3f8b1"}
185322024-02-03T17:30:46.249Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:45.417313771Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823}
185332024-02-03T17:30:46.249ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
185342024-02-03T17:30:46.249ZLoading write count information from file "/tmp/test_repair_verify.data"
185352024-02-03T17:30:46.249ZRun Repair workload
185362024-02-03T17:30:46.249Z01/30 Write block 594 len 20480 data: 10 10 10 9 7
185372024-02-03T17:30:46.249Z02/30 Read block 549 len 36864
185382024-02-03T17:30:46.253Z03/30 Write block 192 len 32768 data: 9 7 6 8 10 9 10 11
185392024-02-03T17:30:46.253Z04/30 Read block 44 len 20480
185402024-02-03T17:30:46.253Z05/30 Flush
185412024-02-03T17:30:46.257Z06/30 Write block 169 len 40960 data: 13 13 13 10 10 12 12 14 16 14
185422024-02-03T17:30:46.257Z07/30 Read block 82 len 8192
185432024-02-03T17:30:46.257Z08/30 Flush
185442024-02-03T17:30:46.261Z09/30 Write block 279 len 20480 data: 15 15 17 19 17
185452024-02-03T17:30:46.261Z10/30 Read block 575 len 36864
185462024-02-03T17:30:46.264Z11/30 Flush
185472024-02-03T17:30:46.267Z12/30 Write block 107 len 12288 data: 19 18 18
185482024-02-03T17:30:46.267Z13/30 Write block 418 len 28672 data: 14 14 17 15 15 15 14
185492024-02-03T17:30:46.267Z14/30 Flush
185502024-02-03T17:30:46.271Z15/30 Write block 508 len 8192 data: 20 20
185512024-02-03T17:30:46.274Z16/30 Write block 321 len 24576 data: 11 12 11 11 9 11
185522024-02-03T17:30:46.274Z17/30 Write block 468 len 32768 data: 12 12 12 13 10 9 11 10
185532024-02-03T17:30:46.278Z18/30 Write block 485 len 28672 data: 9 11 11 12 14 15 14
185542024-02-03T17:30:46.279Z19/30 Write block 109 len 28672 data: 19 18 15 14 17 18 17
185552024-02-03T17:30:46.279Z20/30 Write block 266 len 16384 data: 17 14 16 18
185562024-02-03T17:30:46.279Z21/30 Flush
185572024-02-03T17:30:46.283Z22/30 Write block 454 len 16384 data: 8 11 16 18
185582024-02-03T17:30:46.283Z23/30 Read block 72 len 16384
185592024-02-03T17:30:46.287Z24/30 Write block 495 len 40960 data: 11 14 13 16 18 18 20 17 18 18
185602024-02-03T17:30:46.287Z25/30 Write block 390 len 8192 data: 13 13
185612024-02-03T17:30:46.287Z26/30 Read block 235 len 20480
185622024-02-03T17:30:46.291Z27/30 Write block 34 len 36864 data: 8 7 8 9 12 13 11 9 11
185632024-02-03T17:30:46.291Z28/30 Read block 209 len 28672
185642024-02-03T17:30:46.296Z29/30 Write block 579 len 12288 data: 20 20 21
185652024-02-03T17:30:46.296Z30/30 Write block 199 len 28672 data: 12 9 9 10 12 12 10
185662024-02-03T17:30:46.296Z----------------------------------------------------------------
185672024-02-03T17:30:46.296Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185682024-02-03T17:30:46.296ZStates: Active Active Active
185692024-02-03T17:30:46.296ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185702024-02-03T17:30:46.296Z 1 Acked 1000 Write 5 Sent Done Done false
185712024-02-03T17:30:46.296Z 2 Acked 1001 Read 9 Sent Done Done false
185722024-02-03T17:30:46.296Z 3 Acked 1002 Write 8 Sent Done Done false
185732024-02-03T17:30:46.296Z 4 Acked 1003 Read 5 Sent Done Done false
185742024-02-03T17:30:46.296Z 5 Acked 1004 Flush 0 Sent Done Done false
185752024-02-03T17:30:46.297Z 6 Acked 1005 Write 10 Sent Done Done false
185762024-02-03T17:30:46.297Z 7 Acked 1006 Read 2 Sent Done Done false
185772024-02-03T17:30:46.297Z 8 Acked 1007 Flush 0 Sent Done Done false
185782024-02-03T17:30:46.297Z 9 Acked 1008 Write 5 Sent Done Done false
185792024-02-03T17:30:46.297Z 10 Acked 1009 Read 9 Sent Done Done false
185802024-02-03T17:30:46.297Z 11 Acked 1010 Flush 0 Sent Done Done false
185812024-02-03T17:30:46.297Z 12 Acked 1011 Write 3 Sent Done Done false
185822024-02-03T17:30:46.297Z 13 Acked 1012 Write 7 Sent Done Done false
185832024-02-03T17:30:46.297Z 14 Acked 1013 Flush 0 Sent Done Done false
185842024-02-03T17:30:46.297Z 15 Acked 1014 Write 2 Sent Done Done false
185852024-02-03T17:30:46.297Z 16 Acked 1015 Write 6 Sent Done Done false
185862024-02-03T17:30:46.297Z 17 Acked 1016 Write 8 Sent Done Done false
185872024-02-03T17:30:46.297Z 18 Acked 1017 Write 7 Sent Done Done false
185882024-02-03T17:30:46.297Z 19 Acked 1018 Write 7 Sent Done Done false
185892024-02-03T17:30:46.297Z 20 Acked 1019 Write 4 Sent Done Done false
185902024-02-03T17:30:46.297Z 21 Acked 1020 Flush 0 Sent Done Done false
185912024-02-03T17:30:46.297Z 22 Acked 1021 Write 4 Sent Done Done false
185922024-02-03T17:30:46.298Z 23 Acked 1022 Read 4 Sent Done Done false
185932024-02-03T17:30:46.298Z 24 Acked 1023 Write 10 Sent Done Done false
185942024-02-03T17:30:46.298Z 25 Acked 1024 Write 2 Sent Done Done false
185952024-02-03T17:30:46.298Z 26 Acked 1025 Read 5 Sent Done Done false
185962024-02-03T17:30:46.298Z 27 Acked 1026 Write 9 Sent Done Done false
185972024-02-03T17:30:46.298Z 28 Acked 1027 Read 7 Sent Done Done false
185982024-02-03T17:30:46.298Z 29 Acked 1028 Write 3 Sent Done Done false
185992024-02-03T17:30:46.298Z 30 Acked 1029 Write 7 Sent Sent Sent false
186002024-02-03T17:30:46.298Z STATES DS:0 DS:1 DS:2 TOTAL
186012024-02-03T17:30:46.298Z New 0 0 0 0
186022024-02-03T17:30:46.298Z Sent 30 1 1 32
186032024-02-03T17:30:46.298Z Done 0 29 29 58
186042024-02-03T17:30:46.298Z Skipped 0 0 0 0
186052024-02-03T17:30:46.299Z Error 0 0 0 0
186062024-02-03T17:30:46.299ZLast Flush: 0 1020 1020
186072024-02-03T17:30:46.299ZDownstairs last five completed:
186082024-02-03T17:30:46.299ZUpstairs last five completed: 30 29 28 27 26
186092024-02-03T17:30:46.299Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:45.470562545Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1823}
186102024-02-03T17:30:46.299ZWrote out file "/tmp/test_repair_verify.data"
186112024-02-03T17:30:46.304Z
186122024-02-03T17:30:46.310ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
186132024-02-03T17:30:46.341ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186142024-02-03T17:30:46.342Z 0 000-019 170 170 170 311 311 311 F F F
186152024-02-03T17:30:46.345Z 1 020-039 170 170 170 311 311 311 F T T <---
186162024-02-03T17:30:46.345Z 2 040-059 167 167 167 307 307 307 F T T <---
186172024-02-03T17:30:46.346Z 3 060-079 166 166 166 306 306 306 F F F
186182024-02-03T17:30:46.346Z 4 080-099 166 166 166 306 306 306 F F F
186192024-02-03T17:30:46.346Z 5 100-119 169 172 172 310 316 316 F F F <---
186202024-02-03T17:30:46.346Z 6 120-139 154 154 154 282 282 282 F F F
186212024-02-03T17:30:46.346Z 7 140-159 170 170 170 311 311 311 F F F
186222024-02-03T17:30:46.346Z 8 160-179 168 172 172 309 313 313 F F F <---
186232024-02-03T17:30:46.346Z 9 180-199 170 172 172 311 312 312 F T T <---
186242024-02-03T17:30:46.346Z 10 200-219 163 163 163 298 298 298 F T T <---
186252024-02-03T17:30:46.346Z 11 220-239 167 167 167 307 307 307 F F F
186262024-02-03T17:30:46.346Z 12 240-259 170 170 170 311 311 311 F F F
186272024-02-03T17:30:46.346Z 13 260-279 166 172 172 306 316 316 F F F <---
186282024-02-03T17:30:46.346Z 14 280-299 170 172 172 311 314 314 F F F <---
186292024-02-03T17:30:46.347Z 15 300-319 170 170 170 311 311 311 F F F
186302024-02-03T17:30:46.347Z 16 320-339 159 172 172 292 316 316 F F F <---
186312024-02-03T17:30:46.347Z 17 340-359 168 168 168 308 308 308 F F F
186322024-02-03T17:30:46.347Z 18 360-379 165 165 165 303 303 303 F F F
186332024-02-03T17:30:46.347Z 19 380-399 170 170 170 311 311 311 F T T <---
186342024-02-03T17:30:46.347Z 20 400-419 170 172 172 311 315 315 F F F <---
186352024-02-03T17:30:46.347Z 21 420-439 170 172 172 311 315 315 F F F <---
186362024-02-03T17:30:46.347Z 22 440-459 166 166 166 306 306 306 F T T <---
186372024-02-03T17:30:46.347Z 23 460-479 166 172 172 305 316 316 F F F <---
186382024-02-03T17:30:46.347Z 24 480-499 170 172 172 311 316 316 F T T <---
186392024-02-03T17:30:46.347Z 25 500-519 167 172 172 307 316 316 F T T <---
186402024-02-03T17:30:46.347Z 26 520-539 165 165 165 303 303 303 F F F
186412024-02-03T17:30:46.348Z 27 540-559 166 166 166 306 306 306 F F F
186422024-02-03T17:30:46.348Z 28 560-579 169 169 169 310 310 310 F T T <---
186432024-02-03T17:30:46.348Z 29 580-599 170 172 172 311 312 312 F T T <---
186442024-02-03T17:30:46.348ZMax gen: 172, Max flush: 316
186452024-02-03T17:30:46.348ZError: Difference in extent metadata found!
186462024-02-03T17:30:46.348ZA Difference in extent metadata is expected here
186472024-02-03T17:30:46.348ZOn loop 86
186482024-02-03T17:30:46.348Z
186492024-02-03T17:30:46.352ZVerifying data now
186502024-02-03T17:30:51.862ZNo change after verify
186512024-02-03T17:30:51.862ZLoop: 86 Downstairs dump after verify (and repair):
186522024-02-03T17:30:51.889ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186532024-02-03T17:30:51.889Z 0 000-019 170 170 170 311 311 311 F F F
186542024-02-03T17:30:51.889Z 1 020-039 173 173 173 317 317 317 F F F
186552024-02-03T17:30:51.889Z 2 040-059 173 173 173 317 317 317 F F F
186562024-02-03T17:30:51.889Z 3 060-079 166 166 166 306 306 306 F F F
186572024-02-03T17:30:51.889Z 4 080-099 166 166 166 306 306 306 F F F
186582024-02-03T17:30:51.889Z 5 100-119 172 172 172 316 316 316 F F F
186592024-02-03T17:30:51.896Z 6 120-139 154 154 154 282 282 282 F F F
186602024-02-03T17:30:51.896Z 7 140-159 170 170 170 311 311 311 F F F
186612024-02-03T17:30:51.896Z 8 160-179 172 172 172 313 313 313 F F F
186622024-02-03T17:30:51.896Z 9 180-199 173 173 173 317 317 317 F F F
186632024-02-03T17:30:51.896Z 10 200-219 173 173 173 317 317 317 F F F
186642024-02-03T17:30:51.896Z 11 220-239 167 167 167 307 307 307 F F F
186652024-02-03T17:30:51.896Z 12 240-259 170 170 170 311 311 311 F F F
186662024-02-03T17:30:51.896Z 13 260-279 172 172 172 316 316 316 F F F
186672024-02-03T17:30:51.896Z 14 280-299 172 172 172 314 314 314 F F F
186682024-02-03T17:30:51.896Z 15 300-319 170 170 170 311 311 311 F F F
186692024-02-03T17:30:51.896Z 16 320-339 172 172 172 316 316 316 F F F
186702024-02-03T17:30:51.896Z 17 340-359 168 168 168 308 308 308 F F F
186712024-02-03T17:30:51.896Z 18 360-379 165 165 165 303 303 303 F F F
186722024-02-03T17:30:51.896Z 19 380-399 173 173 173 317 317 317 F F F
186732024-02-03T17:30:51.896Z 20 400-419 172 172 172 315 315 315 F F F
186742024-02-03T17:30:51.896Z 21 420-439 172 172 172 315 315 315 F F F
186752024-02-03T17:30:51.896Z 22 440-459 173 173 173 317 317 317 F F F
186762024-02-03T17:30:51.896Z 23 460-479 172 172 172 316 316 316 F F F
186772024-02-03T17:30:51.897Z 24 480-499 173 173 173 317 317 317 F F F
186782024-02-03T17:30:51.897Z 25 500-519 173 173 173 317 317 317 F F F
186792024-02-03T17:30:51.897Z 26 520-539 165 165 165 303 303 303 F F F
186802024-02-03T17:30:51.897Z 27 540-559 166 166 166 306 306 306 F F F
186812024-02-03T17:30:51.897Z 28 560-579 173 173 173 317 317 317 F F F
186822024-02-03T17:30:51.897Z 29 580-599 173 173 173 317 317 317 F F F
186832024-02-03T17:30:51.897ZMax gen: 173, Max flush: 317
186842024-02-03T17:30:51.897Z
186852024-02-03T17:30:51.897ZBegin loop 87
186862024-02-03T17:30:51.897ZDownstairs to restart: 2
186872024-02-03T17:30:51.928Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.103087873Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831}
186882024-02-03T17:30:51.928Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.103635345Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831}
186892024-02-03T17:30:51.928Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.103696454Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831}
186902024-02-03T17:30:51.928Z{"msg":"Crucible cae53a61-98c6-4a8c-9fca-f480d42ac340 has session id: 7ca47f16-006e-4409-97bd-7871b3a13e78","vCrucible runtime is spawned
186912024-02-03T17:30:51.934Z":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.103780377Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
186922024-02-03T17:30:51.935Z{"msg":"Upstairs opts: Upstairs UUID: cae53a61-98c6-4a8c-9fca-f480d42ac340, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.103906442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
186932024-02-03T17:30:51.935Z{"msg":"Crucible stats registered with UUID: cae53a61-98c6-4a8c-9fca-f480d42ac340","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.103981293Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
186942024-02-03T17:30:51.935Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104021183Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831}
186952024-02-03T17:30:51.935Z{"msg":"cae53a61-98c6-4a8c-9fca-f480d42ac340 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104184405Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
186962024-02-03T17:30:51.935Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104250752Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
186972024-02-03T17:30:51.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104278604Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
186982024-02-03T17:30:51.935Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104301573Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
186992024-02-03T17:30:51.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104339567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187002024-02-03T17:30:51.935Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104399334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187012024-02-03T17:30:51.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104426188Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187022024-02-03T17:30:51.935Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104453767Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187032024-02-03T17:30:51.935Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.10454393Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187042024-02-03T17:30:51.935Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.104579457Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187052024-02-03T17:30:51.935Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.10476933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187062024-02-03T17:30:51.935Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.105068734Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187072024-02-03T17:30:51.935Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.105272737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187082024-02-03T17:30:51.935Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.10598934Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187092024-02-03T17:30:51.935Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.106075359Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187102024-02-03T17:30:51.935Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.106148548Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187112024-02-03T17:30:51.935Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.106193875Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187122024-02-03T17:30:51.935Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.106861607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187132024-02-03T17:30:51.935Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.106921902Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187142024-02-03T17:30:51.936Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.107026661Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187152024-02-03T17:30:51.936Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.107189426Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187162024-02-03T17:30:51.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.107760635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187172024-02-03T17:30:51.936Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.107886304Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187182024-02-03T17:30:51.936Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108032373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187192024-02-03T17:30:51.936Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108120298Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187202024-02-03T17:30:51.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108178061Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187212024-02-03T17:30:51.936Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108203493Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187222024-02-03T17:30:51.936Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108244609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187232024-02-03T17:30:51.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108694915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187242024-02-03T17:30:51.936Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108767753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187252024-02-03T17:30:51.936Z{"msg":"[0]R flush_numbers[0..12]: [311, 317, 317, 306, 306, 316, 282, 311, 313, 317, 317, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108795085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187262024-02-03T17:30:51.936Z{"msg":"[0]R generation[0..12]: [170, 173, 173, 166, 166, 172, 154, 170, 172, 173, 173, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108837885Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187272024-02-03T17:30:51.936Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108864511Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187282024-02-03T17:30:51.936Z{"msg":"[1]R flush_numbers[0..12]: [311, 317, 317, 306, 306, 316, 282, 311, 313, 317, 317, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108907203Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187292024-02-03T17:30:51.936Z{"msg":"[1]R generation[0..12]: [170, 173, 173, 166, 166, 172, 154, 170, 172, 173, 173, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108932982Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187302024-02-03T17:30:51.936Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.108975286Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187312024-02-03T17:30:51.936Z{"msg":"[2]R flush_numbers[0..12]: [311, 317, 317, 306, 306, 316, 282, 311, 313, 317, 317, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109000788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187322024-02-03T17:30:51.936ZWait for a query_work_queue command to finish before sending IO
187332024-02-03T17:30:51.936Z{"msg":"[2]R generation[0..12]: [170, 173, 173, 166, 166, 172, 154, 170, 172, 173, 173, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109066518Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187342024-02-03T17:30:51.936Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109117705Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187352024-02-03T17:30:51.943Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109142611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187362024-02-03T17:30:51.943Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109236772Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187372024-02-03T17:30:51.943Z{"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.10928273Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187382024-02-03T17:30:51.943Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109317679Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187392024-02-03T17:30:51.943Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109346633Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187402024-02-03T17:30:51.943Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109415431Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187412024-02-03T17:30:51.943Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109453625Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187422024-02-03T17:30:51.943Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109493433Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187432024-02-03T17:30:51.943Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109551751Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187442024-02-03T17:30:51.943Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109576554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187452024-02-03T17:30:51.943Z{"msg":"cae53a61-98c6-4a8c-9fca-f480d42ac340 is now active with session: 7ca47f16-006e-4409-97bd-7871b3a13e78","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109644588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187462024-02-03T17:30:51.943Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187472024-02-03T17:30:51.943Z,"time":"2024-02-03T17:30:51.109707048Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831,"session_id":"7ca47f16-006e-4409-97bd-7871b3a13e78"}
187482024-02-03T17:30:51.943Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:51.109777248Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831}
187492024-02-03T17:30:51.943ZLoading write count information from file "/tmp/test_repair_verify.data"
187502024-02-03T17:30:51.943ZRun Repair workload
187512024-02-03T17:30:51.943Z01/30 Write block 66 len 20480 data: 11 9 7 8 8
187522024-02-03T17:30:51.944Z02/30 Write block 154 len 24576 data: 12 15 15 15 15 14
187532024-02-03T17:30:51.944Z03/30 Read block 466 len 16384
187542024-02-03T17:30:51.948Z04/30 Write block 1 len 28672 data: 4 10 11 13 15 15 16
187552024-02-03T17:30:51.948Z05/30 Write block 436 len 4096 data: 14
187562024-02-03T17:30:51.948Z06/30 Read block 395 len 4096
187572024-02-03T17:30:51.948Z07/30 Read block 177 len 12288
187582024-02-03T17:30:51.948Z08/30 Flush
187592024-02-03T17:30:51.948Z09/30 Write block 182 len 8192 data: 13 11
187602024-02-03T17:30:51.948Z10/30 Read block 536 len 8192
187612024-02-03T17:30:51.953Z11/30 Write block 472 len 20480 data: 11 10 12 11 11
187622024-02-03T17:30:51.953Z12/30 Read block 414 len 4096
187632024-02-03T17:30:51.953Z13/30 Write block 197 len 40960 data: 10 11 13 10 10 11 13 13 11 9
187642024-02-03T17:30:51.957Z14/30 Read block 416 len 24576
187652024-02-03T17:30:51.957Z15/30 Read block 439 len 36864
187662024-02-03T17:30:51.957Z16/30 Read block 59 len 16384
187672024-02-03T17:30:51.963Z17/30 Write block 410 len 24576 data: 12 12 11 14 11 7
187682024-02-03T17:30:51.963Z18/30 Write block 536 len 12288 data: 12 12 11
187692024-02-03T17:30:51.967Z19/30 Write block 378 len 24576 data: 13 11 12 13 12 11
187702024-02-03T17:30:51.967Z20/30 Write block 248 len 28672 data: 13 15 20 13 10 12 14
187712024-02-03T17:30:51.967Z21/30 Write block 127 len 32768 data: 12 16 18 19 18 18 13 16
187722024-02-03T17:30:51.967Z22/30 Read block 491 len 36864
187732024-02-03T17:30:51.967Z23/30 Write block 95 len 8192 data: 11 12
187742024-02-03T17:30:51.971Z24/30 Read block 582 len 28672
187752024-02-03T17:30:51.971Z25/30 Write block 518 len 8192 data: 13 10
187762024-02-03T17:30:51.971Z26/30 Flush
187772024-02-03T17:30:51.984Z27/30 Write block 114 len 12288 data: 19 18 18
187782024-02-03T17:30:52.011Z28/30 Read block 201 len 32768
187792024-02-03T17:30:52.011Z29/30 Read block 294 len 40960
187802024-02-03T17:30:52.011Z30/30 Read block 240 len 4096
187812024-02-03T17:30:52.011Z----------------------------------------------------------------
187822024-02-03T17:30:52.011Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
187832024-02-03T17:30:52.011ZStates: Active Active Active
187842024-02-03T17:30:52.011ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187852024-02-03T17:30:52.011Z 1 Acked 1000 Write 5 Done Done Done false
187862024-02-03T17:30:52.012Z 2 Acked 1001 Write 6 Done Done Sent false
187872024-02-03T17:30:52.012Z 3 Acked 1002 Read 4 Done Done Sent false
187882024-02-03T17:30:52.012Z 4 Acked 1003 Write 7 Done Done Sent false
187892024-02-03T17:30:52.012Z 5 Acked 1004 Write 1 Done Done Sent false
187902024-02-03T17:30:52.012Z 6 Acked 1005 Read 1 Done Done Sent false
187912024-02-03T17:30:52.012Z 7 Acked 1006 Read 3 Done Done Sent false
187922024-02-03T17:30:52.012Z 8 Acked 1007 Flush 0 Done Done Sent false
187932024-02-03T17:30:52.012Z 9 Acked 1008 Write 2 Done Done Sent false
187942024-02-03T17:30:52.012Z 10 Acked 1009 Read 2 Done Done Sent false
187952024-02-03T17:30:52.012Z 11 Acked 1010 Write 5 Done Done Sent false
187962024-02-03T17:30:52.012Z 12 Acked 1011 Read 1 Done Done Sent false
187972024-02-03T17:30:52.012Z 13 Acked 1012 Write 10 Done Done Sent false
187982024-02-03T17:30:52.012Z 14 Acked 1013 Read 6 Done Done Sent false
187992024-02-03T17:30:52.013Z 15 Acked 1014 Read 9 Done Done Sent false
188002024-02-03T17:30:52.013Z 16 Acked 1015 Read 4 Done Done Sent false
188012024-02-03T17:30:52.013Z 17 Acked 1016 Write 6 Done Done Sent false
188022024-02-03T17:30:52.013Z 18 Acked 1017 Write 3 Done Done Sent false
188032024-02-03T17:30:52.013Z 19 Acked 1018 Write 6 Done Done Sent false
188042024-02-03T17:30:52.013Z 20 Acked 1019 Write 7 Done Done Sent false
188052024-02-03T17:30:52.013Z 21 Acked 1020 Write 8 Done Done Sent false
188062024-02-03T17:30:52.013Z 22 Acked 1021 Read 9 Done Done Sent false
188072024-02-03T17:30:52.013Z 23 Acked 1022 Write 2 Done Done Sent false
188082024-02-03T17:30:52.013Z 24 Acked 1023 Read 7 Done Done Sent false
188092024-02-03T17:30:52.013Z 25 Acked 1024 Write 2 Done Done Sent false
188102024-02-03T17:30:52.013Z 26 Acked 1025 Flush 0 Done Done Sent false
188112024-02-03T17:30:52.013Z 27 Acked 1026 Write 3 Done Done Sent false
188122024-02-03T17:30:52.013Z 28 Acked 1027 Read 8 Done Done Sent false
188132024-02-03T17:30:52.014Z 29 Acked 1028 Read 10 Done Done Sent false
188142024-02-03T17:30:52.014Z 30 Acked 1029 Read 1 Done Done Sent false
188152024-02-03T17:30:52.014Z STATES DS:0 DS:1 DS:2 TOTAL
188162024-02-03T17:30:52.014Z New 0 0 0 0
188172024-02-03T17:30:52.014Z Sent 0 0 29 29
188182024-02-03T17:30:52.014Z Done 30 30 1 61
188192024-02-03T17:30:52.014Z Skipped 0 0 0 0
188202024-02-03T17:30:52.014Z Error 0 0 0 0
188212024-02-03T17:30:52.014ZLast Flush: 1025 1025 0
188222024-02-03T17:30:52.014ZDownstairs last five completed:
188232024-02-03T17:30:52.014ZUpstairs last five completed: 30 29 28 27 26
188242024-02-03T17:30:52.014Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:51.159005514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1831}
188252024-02-03T17:30:52.014ZWrote out file "/tmp/test_repair_verify.data"
188262024-02-03T17:30:52.014Z
188272024-02-03T17:30:52.019ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188282024-02-03T17:30:52.022ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188292024-02-03T17:30:52.022Z 0 000-019 174 174 170 318 318 311 F F F <---
188302024-02-03T17:30:52.022Z 1 020-039 173 173 173 317 317 317 F F F
188312024-02-03T17:30:52.022Z 2 040-059 173 173 173 317 317 317 F F F
188322024-02-03T17:30:52.022Z 3 060-079 174 174 166 318 318 306 F F T <---
188332024-02-03T17:30:52.023Z 4 080-099 174 174 166 319 319 306 F F F <---
188342024-02-03T17:30:52.023Z 5 100-119 172 172 172 316 316 316 T T F <---
188352024-02-03T17:30:52.023Z 6 120-139 174 174 154 319 319 282 F F F <---
188362024-02-03T17:30:52.023Z 7 140-159 174 174 170 318 318 311 F F F <---
188372024-02-03T17:30:52.023Z 8 160-179 172 172 172 313 313 313 F F F
188382024-02-03T17:30:52.023Z 9 180-199 174 174 173 319 319 317 F F F <---
188392024-02-03T17:30:52.023Z 10 200-219 174 174 173 319 319 317 F F F <---
188402024-02-03T17:30:52.027Z 11 220-239 167 167 167 307 307 307 F F F
188412024-02-03T17:30:52.027Z 12 240-259 174 174 170 319 319 311 F F F <---
188422024-02-03T17:30:52.027Z 13 260-279 172 172 172 316 316 316 F F F
188432024-02-03T17:30:52.027Z 14 280-299 172 172 172 314 314 314 F F F
188442024-02-03T17:30:52.027Z 15 300-319 170 170 170 311 311 311 F F F
188452024-02-03T17:30:52.027Z 16 320-339 172 172 172 316 316 316 F F F
188462024-02-03T17:30:52.027Z 17 340-359 168 168 168 308 308 308 F F F
188472024-02-03T17:30:52.027Z 18 360-379 174 174 165 319 319 303 F F F <---
188482024-02-03T17:30:52.027Z 19 380-399 174 174 173 319 319 317 F F F <---
188492024-02-03T17:30:52.027Z 20 400-419 174 174 172 319 319 315 F F F <---
188502024-02-03T17:30:52.027Z 21 420-439 174 174 172 318 318 315 F F F <---
188512024-02-03T17:30:52.028Z 22 440-459 173 173 173 317 317 317 F F F
188522024-02-03T17:30:52.028Z 23 460-479 174 174 172 319 319 316 F F F <---
188532024-02-03T17:30:52.028Z 24 480-499 173 173 173 317 317 317 F F F
188542024-02-03T17:30:52.028Z 25 500-519 174 174 173 319 319 317 F F F <---
188552024-02-03T17:30:52.028Z 26 520-539 174 174 165 319 319 303 F F F <---
188562024-02-03T17:30:52.028Z 27 540-559 166 166 166 306 306 306 F F F
188572024-02-03T17:30:52.028Z 28 560-579 173 173 173 317 317 317 F F F
188582024-02-03T17:30:52.028Z 29 580-599 173 173 173 317 317 317 F F F
188592024-02-03T17:30:52.028ZMax gen: 174, Max flush: 319
188602024-02-03T17:30:52.028ZError: Difference in extent metadata found!
188612024-02-03T17:30:52.032ZA Difference in extent metadata is expected here
188622024-02-03T17:30:52.032ZOn loop 87
188632024-02-03T17:30:52.032Z
188642024-02-03T17:30:52.032ZVerifying data now
188652024-02-03T17:30:57.358ZNo change after verify
188662024-02-03T17:30:57.358ZLoop: 87 Downstairs dump after verify (and repair):
188672024-02-03T17:30:57.383ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188682024-02-03T17:30:57.383Z 0 000-019 174 174 174 318 318 318 F F F
188692024-02-03T17:30:57.386Z 1 020-039 173 173 173 317 317 317 F F F
188702024-02-03T17:30:57.386Z 2 040-059 173 173 173 317 317 317 F F F
188712024-02-03T17:30:57.386Z 3 060-079 174 174 174 318 318 318 F F F
188722024-02-03T17:30:57.386Z 4 080-099 174 174 174 319 319 319 F F F
188732024-02-03T17:30:57.386Z 5 100-119 175 175 175 320 320 320 F F F
188742024-02-03T17:30:57.386Z 6 120-139 174 174 174 319 319 319 F F F
188752024-02-03T17:30:57.386Z 7 140-159 174 174 174 318 318 318 F F F
188762024-02-03T17:30:57.386Z 8 160-179 172 172 172 313 313 313 F F F
188772024-02-03T17:30:57.387Z 9 180-199 174 174 174 319 319 319 F F F
188782024-02-03T17:30:57.387Z 10 200-219 174 174 174 319 319 319 F F F
188792024-02-03T17:30:57.387Z 11 220-239 167 167 167 307 307 307 F F F
188802024-02-03T17:30:57.387Z 12 240-259 174 174 174 319 319 319 F F F
188812024-02-03T17:30:57.387Z 13 260-279 172 172 172 316 316 316 F F F
188822024-02-03T17:30:57.387Z 14 280-299 172 172 172 314 314 314 F F F
188832024-02-03T17:30:57.387Z 15 300-319 170 170 170 311 311 311 F F F
188842024-02-03T17:30:57.387Z 16 320-339 172 172 172 316 316 316 F F F
188852024-02-03T17:30:57.387Z 17 340-359 168 168 168 308 308 308 F F F
188862024-02-03T17:30:57.387Z 18 360-379 174 174 174 319 319 319 F F F
188872024-02-03T17:30:57.387Z 19 380-399 174 174 174 319 319 319 F F F
188882024-02-03T17:30:57.387Z 20 400-419 174 174 174 319 319 319 F F F
188892024-02-03T17:30:57.388Z 21 420-439 174 174 174 318 318 318 F F F
188902024-02-03T17:30:57.388Z 22 440-459 173 173 173 317 317 317 F F F
188912024-02-03T17:30:57.388Z 23 460-479 174 174 174 319 319 319 F F F
188922024-02-03T17:30:57.388Z 24 480-499 173 173 173 317 317 317 F F F
188932024-02-03T17:30:57.388Z 25 500-519 174 174 174 319 319 319 F F F
188942024-02-03T17:30:57.388Z 26 520-539 174 174 174 319 319 319 F F F
188952024-02-03T17:30:57.388Z 27 540-559 166 166 166 306 306 306 F F F
188962024-02-03T17:30:57.388Z 28 560-579 173 173 173 317 317 317 F F F
188972024-02-03T17:30:57.388Z 29 580-599 173 173 173 317 317 317 F F F
188982024-02-03T17:30:57.388ZMax gen: 175, Max flush: 320
188992024-02-03T17:30:57.392Z
189002024-02-03T17:30:57.392ZBegin loop 88
189012024-02-03T17:30:57.392ZDownstairs to restart: 2
189022024-02-03T17:30:57.419Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.596089062Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839}
189032024-02-03T17:30:57.419Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.596595985Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839}
189042024-02-03T17:30:57.423Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.596677119Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839}
189052024-02-03T17:30:57.423Z{"msg":"Crucible bd4d6300-cf23-41b1-a9c6-11d8366fb334 has session id: cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.596757904Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189062024-02-03T17:30:57.424Z{"msg":"Upstairs opts: Upstairs UUID: bd4d6300-cf23-41b1-a9c6-11d8366fb334, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.59681511Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189072024-02-03T17:30:57.424Z{"msg":"Crucible stats registered with UUID: bd4d6300-cf23-41b1-a9c6-11d8366fb334","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.596892632Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"Crucible runtime is spawned
189082024-02-03T17:30:57.424Zsession_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189092024-02-03T17:30:57.424Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597083141Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839}
189102024-02-03T17:30:57.424Z{"msg":"bd4d6300-cf23-41b1-a9c6-11d8366fb334 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597145822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189112024-02-03T17:30:57.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597170305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189122024-02-03T17:30:57.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597211848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189132024-02-03T17:30:57.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597309205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189142024-02-03T17:30:57.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597436914Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189152024-02-03T17:30:57.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597552508Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189162024-02-03T17:30:57.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597637115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189172024-02-03T17:30:57.424Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597853437Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189182024-02-03T17:30:57.425Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.597919007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189192024-02-03T17:30:57.425Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.598041185Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189202024-02-03T17:30:57.425Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.598112128Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189212024-02-03T17:30:57.425Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.598182235Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189222024-02-03T17:30:57.425Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.598228815Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189232024-02-03T17:30:57.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.598882106Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189242024-02-03T17:30:57.425Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.59895016Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189252024-02-03T17:30:57.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.598994921Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189262024-02-03T17:30:57.425Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.599139906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189272024-02-03T17:30:57.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.599218347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189282024-02-03T17:30:57.425Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.599263803Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189292024-02-03T17:30:57.430Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.600088309Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189302024-02-03T17:30:57.430Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.600161298Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189312024-02-03T17:30:57.430Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.60020207Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189322024-02-03T17:30:57.430Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601046748Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189332024-02-03T17:30:57.430Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.60118013Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189342024-02-03T17:30:57.430Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601212915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189352024-02-03T17:30:57.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601254919Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189362024-02-03T17:30:57.431Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601279561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189372024-02-03T17:30:57.431Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601320796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189382024-02-03T17:30:57.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601343283Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189392024-02-03T17:30:57.431Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601365674Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189402024-02-03T17:30:57.431Z{"msg":"[0]R flush_numbers[0..12]: [318, 317, 317, 318, 319, 320, 319, 318, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601389206Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189412024-02-03T17:30:57.431Z{"msg":"[0]R generation[0..12]: [174, 173, 173, 174, 174, 175, 174, 174, 172, 174, 174, 167]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
189422024-02-03T17:30:57.431Z,"time":"2024-02-03T17:30:56.601430973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189432024-02-03T17:30:57.431Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601457034Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189442024-02-03T17:30:57.431Z{"msg":"[1]R flush_numbers[0..12]: [318, 317, 317, 318, 319, 320, 319, 318, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601507566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189452024-02-03T17:30:57.431Z{"msg":"[1]R generation[0..12]: [174, 173, 173, 174, 174, 175, 174, 174, 172, 174, 174, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601531875Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189462024-02-03T17:30:57.431Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601592682Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189472024-02-03T17:30:57.432Z{"msg":"[2]R flush_numbers[0..12]: [318, 317, 317, 318, 319, 320, 319, 318, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601622822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189482024-02-03T17:30:57.432Z{"msg":"[2]R generation[0..12]: [174, 173, 173, 174, 174, 175, 174, 174, 172, 174, 174, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601647578Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189492024-02-03T17:30:57.432Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601691296Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189502024-02-03T17:30:57.432Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601714971Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189512024-02-03T17:30:57.432Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601756555Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189522024-02-03T17:30:57.432Z{"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601818939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189532024-02-03T17:30:57.432Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601846132Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189542024-02-03T17:30:57.432Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.60188862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189552024-02-03T17:30:57.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601910676Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189562024-02-03T17:30:57.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.601971976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189572024-02-03T17:30:57.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.602030397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189582024-02-03T17:30:57.432Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.602056063Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189592024-02-03T17:30:57.432Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.602102543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189602024-02-03T17:30:57.433Z{"msg":"bd4d6300-cf23-41b1-a9c6-11d8366fb334 is now active with session: cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.602126047Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189612024-02-03T17:30:57.433Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.602178332Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839,"session_id":"cc3fc1d6-d3db-4696-bc2c-1b21cc54c76f"}
189622024-02-03T17:30:57.433Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:30:56.60220713Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839}
189632024-02-03T17:30:57.433ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189642024-02-03T17:30:57.433ZLoading write count information from file "/tmp/test_repair_verify.data"
189652024-02-03T17:30:57.433ZRun Repair workload
189662024-02-03T17:30:57.433Z01/30 Write block 294 len 32768 data: 14 12 14 15 11 8 10 9
189672024-02-03T17:30:57.433Z02/30 Read block 332 len 4096
189682024-02-03T17:30:57.449Z03/30 Write block 575 len 4096 data: 12
189692024-02-03T17:30:57.449Z04/30 Write block 267 len 12288 data: 15 17 19
189702024-02-03T17:30:57.449Z05/30 Write block 496 len 36864 data: 15 14 17 19 19 21 18 19 19
189712024-02-03T17:30:57.449Z06/30 Write block 391 len 28672 data: 14 10 10 11 13 14 11
189722024-02-03T17:30:57.449Z07/30 Read block 71 len 4096
189732024-02-03T17:30:57.449Z08/30 Write block 254 len 40960 data: 15 14 14 12 10 10 14 15 15 15
189742024-02-03T17:30:57.453Z09/30 Read block 35 len 20480
189752024-02-03T17:30:57.453Z10/30 Write block 558 len 24576 data: 15 16 12 11 8 9
189762024-02-03T17:30:57.453Z11/30 Flush
189772024-02-03T17:30:57.453Z12/30 Write block 119 len 36864 data: 16 15 16 12 13 10 11 14 13
189782024-02-03T17:30:57.453Z13/30 Read block 44 len 8192
189792024-02-03T17:30:57.453Z14/30 Read block 28 len 4096
189802024-02-03T17:30:57.453Z15/30 Write block 420 len 28672 data: 18 16 16 16 15 12 11
189812024-02-03T17:30:57.453Z16/30 Read block 205 len 4096
189822024-02-03T17:30:57.454Z17/30 Write block 431 len 16384 data: 14 14 12 15
189832024-02-03T17:30:57.457Z18/30 Read block 439 len 12288
189842024-02-03T17:30:57.457Z19/30 Write block 383 len 32768 data: 12 11 10 8 9 10 11 14
189852024-02-03T17:30:57.457Z20/30 Write block 10 len 36864 data: 20 17 18 15 13 12 12 10 9
189862024-02-03T17:30:57.461Z21/30 Flush
189872024-02-03T17:30:57.465Z22/30 Write block 9 len 28672 data: 17 21 18 19 16 14 13
189882024-02-03T17:30:57.465Z23/30 Read block 316 len 40960
189892024-02-03T17:30:57.468Z24/30 Write block 441 len 36864 data: 16 12 12 12 14 13 11 12 9
189902024-02-03T17:30:57.468Z25/30 Read block 514 len 8192
189912024-02-03T17:30:57.472Z26/30 Read block 168 len 8192
189922024-02-03T17:30:57.472Z27/30 Read block 307 len 4096
189932024-02-03T17:30:57.472Z28/30 Write block 321 len 12288 data: 12 13 12
189942024-02-03T17:30:57.472Z29/30 Read block 378 len 24576
189952024-02-03T17:30:57.476Z30/30 Write block 140 len 20480 data: 15 14 12 9 10
189962024-02-03T17:30:57.477Z----------------------------------------------------------------
189972024-02-03T17:30:57.477Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
189982024-02-03T17:30:57.477ZStates: Active Active Active
189992024-02-03T17:30:57.477ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
190002024-02-03T17:30:57.477Z 1 Acked 1000 Write 8 Done Done Done false
190012024-02-03T17:30:57.477Z 2 Acked 1001 Read 1 Done Done Done false
190022024-02-03T17:30:57.477Z 3 Acked 1002 Write 1 Done Done Done false
190032024-02-03T17:30:57.477Z 4 Acked 1003 Write 3 Done Done Done false
190042024-02-03T17:30:57.477Z 5 Acked 1004 Write 9 Done Done Done false
190052024-02-03T17:30:57.477Z 6 Acked 1005 Write 7 Done Done Done false
190062024-02-03T17:30:57.477Z 7 Acked 1006 Read 1 Done Done Done false
190072024-02-03T17:30:57.477Z 8 Acked 1007 Write 10 Done Done Done false
190082024-02-03T17:30:57.477Z 9 Acked 1008 Read 5 Done Done Done false
190092024-02-03T17:30:57.478Z 10 Acked 1009 Write 6 Done Done Sent false
190102024-02-03T17:30:57.478Z 11 Acked 1010 Flush 0 Done Done Sent false
190112024-02-03T17:30:57.478Z 12 Acked 1011 Write 9 Done Done Sent false
190122024-02-03T17:30:57.478Z 13 Acked 1012 Read 2 Done Done Sent false
190132024-02-03T17:30:57.478Z 14 Acked 1013 Read 1 Done Done Sent false
190142024-02-03T17:30:57.478Z 15 Acked 1014 Write 7 Done Done Sent false
190152024-02-03T17:30:57.478Z 16 Acked 1015 Read 1 Done Done Sent false
190162024-02-03T17:30:57.478Z 17 Acked 1016 Write 4 Done Done Sent false
190172024-02-03T17:30:57.478Z 18 Acked 1017 Read 3 Done Done Sent false
190182024-02-03T17:30:57.478Z 19 Acked 1018 Write 8 Done Done Sent false
190192024-02-03T17:30:57.478Z 20 Acked 1019 Write 9 Done Done Sent false
190202024-02-03T17:30:57.478Z 21 Acked 1020 Flush 0 Done Done Sent false
190212024-02-03T17:30:57.478Z 22 Acked 1021 Write 7 Done Done Sent false
190222024-02-03T17:30:57.479Z 23 Acked 1022 Read 10 Done Done Sent false
190232024-02-03T17:30:57.479Z 24 Acked 1023 Write 9 Done Done Sent false
190242024-02-03T17:30:57.479Z 25 Acked 1024 Read 2 Done Done Sent false
190252024-02-03T17:30:57.479Z 26 Acked 1025 Read 2 Done Done Sent false
190262024-02-03T17:30:57.479Z 27 Acked 1026 Read 1 Done Done Sent false
190272024-02-03T17:30:57.479Z 28 Acked 1027 Write 3 Done Done Sent false
190282024-02-03T17:30:57.479Z 29 Acked 1028 Read 6 Done Done Sent false
190292024-02-03T17:30:57.479Z 30 Acked 1029 Write 5 Sent Sent Sent false
190302024-02-03T17:30:57.479Z STATES DS:0 DS:1 DS:2 TOTAL
190312024-02-03T17:30:57.479Z New 0 0 0 0
190322024-02-03T17:30:57.479Z Sent 1 1 21 23
190332024-02-03T17:30:57.479Z Done 29 29 9 67
190342024-02-03T17:30:57.480Z Skipped 0 0 0 0
190352024-02-03T17:30:57.480Z Error 0 0 0 0
190362024-02-03T17:30:57.480ZLast Flush: 1020 1020 0
190372024-02-03T17:30:57.480ZDownstairs last five completed:
190382024-02-03T17:30:57.480ZUpstairs last five completed: 30 29 28 27 26
190392024-02-03T17:30:57.480Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:30:56.651743905Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1839}
190402024-02-03T17:30:57.480ZWrote out file "/tmp/test_repair_verify.data"
190412024-02-03T17:30:57.484Z
190422024-02-03T17:30:57.489ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190432024-02-03T17:30:57.517ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190442024-02-03T17:30:57.517Z 0 000-019 176 176 174 322 322 318 T T F <---
190452024-02-03T17:30:57.522Z 1 020-039 173 173 173 317 317 317 F F F
190462024-02-03T17:30:57.522Z 2 040-059 173 173 173 317 317 317 F F F
190472024-02-03T17:30:57.522Z 3 060-079 174 174 174 318 318 318 F F F
190482024-02-03T17:30:57.522Z 4 080-099 174 174 174 319 319 319 F F F
190492024-02-03T17:30:57.523Z 5 100-119 176 176 175 322 322 320 F F F <---
190502024-02-03T17:30:57.523Z 6 120-139 176 176 174 322 322 319 F F F <---
190512024-02-03T17:30:57.523Z 7 140-159 174 174 174 318 318 318 T T F <---
190522024-02-03T17:30:57.523Z 8 160-179 172 172 172 313 313 313 F F F
190532024-02-03T17:30:57.523Z 9 180-199 174 174 174 319 319 319 F F F
190542024-02-03T17:30:57.523Z 10 200-219 174 174 174 319 319 319 F F F
190552024-02-03T17:30:57.523Z 11 220-239 167 167 167 307 307 307 F F F
190562024-02-03T17:30:57.523Z 12 240-259 176 176 174 321 321 319 F F T <---
190572024-02-03T17:30:57.523Z 13 260-279 176 176 172 321 321 316 F F T <---
190582024-02-03T17:30:57.523Z 14 280-299 176 176 172 321 321 314 F F T <---
190592024-02-03T17:30:57.523Z 15 300-319 176 176 170 321 321 311 F F T <---
190602024-02-03T17:30:57.523Z 16 320-339 172 172 172 316 316 316 T T F <---
190612024-02-03T17:30:57.523Z 17 340-359 168 168 168 308 308 308 F F F
190622024-02-03T17:30:57.523Z 18 360-379 174 174 174 319 319 319 F F F
190632024-02-03T17:30:57.524Z 19 380-399 176 176 174 322 322 319 F F T <---
190642024-02-03T17:30:57.524Z 20 400-419 174 174 174 319 319 319 F F F
190652024-02-03T17:30:57.524Z 21 420-439 176 176 174 322 322 318 F F F <---
190662024-02-03T17:30:57.524Z 22 440-459 173 173 173 317 317 317 T T F <---
190672024-02-03T17:30:57.524Z 23 460-479 174 174 174 319 319 319 F F F
190682024-02-03T17:30:57.524Z 24 480-499 176 176 173 321 321 317 F F T <---
190692024-02-03T17:30:57.524Z 25 500-519 176 176 174 321 321 319 F F T <---
190702024-02-03T17:30:57.524Z 26 520-539 174 174 174 319 319 319 F F F
190712024-02-03T17:30:57.524Z 27 540-559 176 176 166 321 321 306 F F F <---
190722024-02-03T17:30:57.524Z 28 560-579 176 176 173 321 321 317 F F T <---
190732024-02-03T17:30:57.524Z 29 580-599 173 173 173 317 317 317 F F F
190742024-02-03T17:30:57.524ZMax gen: 176, Max flush: 322
190752024-02-03T17:30:57.524ZError: Difference in extent metadata found!
190762024-02-03T17:30:57.528ZA Difference in extent metadata is expected here
190772024-02-03T17:30:57.528ZOn loop 88
190782024-02-03T17:30:57.528Z
190792024-02-03T17:30:57.528ZVerifying data now
190802024-02-03T17:31:03.898ZNo change after verify
190812024-02-03T17:31:03.898ZLoop: 88 Downstairs dump after verify (and repair):
190822024-02-03T17:31:03.913ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190832024-02-03T17:31:03.913Z 0 000-019 177 177 177 323 323 323 F F F
190842024-02-03T17:31:03.919Z 1 020-039 173 173 173 317 317 317 F F F
190852024-02-03T17:31:03.919Z 2 040-059 173 173 173 317 317 317 F F F
190862024-02-03T17:31:03.919Z 3 060-079 174 174 174 318 318 318 F F F
190872024-02-03T17:31:03.919Z 4 080-099 174 174 174 319 319 319 F F F
190882024-02-03T17:31:03.919Z 5 100-119 176 176 176 322 322 322 F F F
190892024-02-03T17:31:03.919Z 6 120-139 176 176 176 322 322 322 F F F
190902024-02-03T17:31:03.919Z 7 140-159 177 177 177 323 323 323 F F F
190912024-02-03T17:31:03.919Z 8 160-179 172 172 172 313 313 313 F F F
190922024-02-03T17:31:03.919Z 9 180-199 174 174 174 319 319 319 F F F
190932024-02-03T17:31:03.919Z 10 200-219 174 174 174 319 319 319 F F F
190942024-02-03T17:31:03.919Z 11 220-239 167 167 167 307 307 307 F F F
190952024-02-03T17:31:03.919Z 12 240-259 176 176 176 321 321 321 F F F
190962024-02-03T17:31:03.919Z 13 260-279 176 176 176 321 321 321 F F F
190972024-02-03T17:31:03.919Z 14 280-299 176 176 176 321 321 321 F F F
190982024-02-03T17:31:03.919Z 15 300-319 176 176 176 321 321 321 F F F
190992024-02-03T17:31:03.919Z 16 320-339 177 177 177 323 323 323 F F F
191002024-02-03T17:31:03.920Z 17 340-359 168 168 168 308 308 308 F F F
191012024-02-03T17:31:03.920Z 18 360-379 174 174 174 319 319 319 F F F
191022024-02-03T17:31:03.920Z 19 380-399 176 176 176 322 322 322 F F F
191032024-02-03T17:31:03.920Z 20 400-419 174 174 174 319 319 319 F F F
191042024-02-03T17:31:03.920Z 21 420-439 176 176 176 322 322 322 F F F
191052024-02-03T17:31:03.920Z 22 440-459 177 177 177 323 323 323 F F F
191062024-02-03T17:31:03.920Z 23 460-479 174 174 174 319 319 319 F F F
191072024-02-03T17:31:03.920Z 24 480-499 176 176 176 321 321 321 F F F
191082024-02-03T17:31:03.920Z 25 500-519 176 176 176 321 321 321 F F F
191092024-02-03T17:31:03.920Z 26 520-539 174 174 174 319 319 319 F F F
191102024-02-03T17:31:03.920Z 27 540-559 176 176 176 321 321 321 F F F
191112024-02-03T17:31:03.920Z 28 560-579 176 176 176 321 321 321 F F F
191122024-02-03T17:31:03.920Z 29 580-599 173 173 173 317 317 317 F F F
191132024-02-03T17:31:03.920ZMax gen: 177, Max flush: 323
191142024-02-03T17:31:03.920Z
191152024-02-03T17:31:03.920ZBegin loop 89
191162024-02-03T17:31:03.920ZDownstairs to restart: 2
191172024-02-03T17:31:03.950Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.126650227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847}
191182024-02-03T17:31:03.950Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127206365Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847}
191192024-02-03T17:31:03.955Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.12726626Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847}
191202024-02-03T17:31:03.955Z{"msg":"Crucible e3604432-ccc1-430f-819c-7bb62d943654 has session id: 12e53c2e-480d-47d7-ae1c-9a8181ce2548","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127333382Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847Crucible runtime is spawned
191212024-02-03T17:31:03.955Z,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191222024-02-03T17:31:03.955Z{"msg":"Upstairs opts: Upstairs UUID: e3604432-ccc1-430f-819c-7bb62d943654, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127603027Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191232024-02-03T17:31:03.955Z{"msg":"Crucible stats registered with UUID: e3604432-ccc1-430f-819c-7bb62d943654","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127668219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191242024-02-03T17:31:03.955Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127693992Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847}
191252024-02-03T17:31:03.955Z{"msg":"e3604432-ccc1-430f-819c-7bb62d943654 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127732327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191262024-02-03T17:31:03.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127779784Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191272024-02-03T17:31:03.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127863122Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191282024-02-03T17:31:03.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127932397Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191292024-02-03T17:31:03.956Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127960695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191302024-02-03T17:31:03.956Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.127994832Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191312024-02-03T17:31:03.956Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.12811077Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191322024-02-03T17:31:03.956Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.128180831Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191332024-02-03T17:31:03.956Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.128226695Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191342024-02-03T17:31:03.956Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.128290546Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191352024-02-03T17:31:03.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.128427417Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191362024-02-03T17:31:03.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.128568639Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191372024-02-03T17:31:03.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.128606709Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191382024-02-03T17:31:03.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.129820036Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191392024-02-03T17:31:03.956Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.129881764Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191402024-02-03T17:31:03.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.130000067Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191412024-02-03T17:31:03.956Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.130077265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191422024-02-03T17:31:03.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.130163741Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191432024-02-03T17:31:03.956Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.130199648Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191442024-02-03T17:31:03.956Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.131000351Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191452024-02-03T17:31:03.961Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.131278369Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191462024-02-03T17:31:03.962Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.131372148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191472024-02-03T17:31:03.962Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132164816Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191482024-02-03T17:31:03.962Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.13224437Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191492024-02-03T17:31:03.962Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132280919Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191502024-02-03T17:31:03.962Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132369976Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191512024-02-03T17:31:03.962Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132411347Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191522024-02-03T17:31:03.962Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132476684Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191532024-02-03T17:31:03.962Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132635107Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191542024-02-03T17:31:03.962Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132687488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191552024-02-03T17:31:03.962Z{"msg":"[0]R flush_numbers[0..12]: [323, 317, 317, 318, 319, 322, 322, 323, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132714016Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191562024-02-03T17:31:03.962Z{"msg":"[0]R generation[0..12]: [177, 173, 173, 174, 174, 176, 176, 177, 172, 174, 174, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132757066Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191572024-02-03T17:31:03.962Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132783207Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191582024-02-03T17:31:03.962Z{"msg":"[1]R flush_numbers[0..12]: [323, 317, 317, 318, 319, 322, 322, 323, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132825883Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191592024-02-03T17:31:03.962Z{"msg":"[1]R generation[0..12]: [177, 173, 173, 174, 174, 176, 176, 177, 172, 174, 174, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132850939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191602024-02-03T17:31:03.962Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132893805Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191612024-02-03T17:31:03.962Z{"msg":"[2]R flush_numbers[0..12]: [323, 317, 317, 318, 319, 322, 322, 323, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132918924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191622024-02-03T17:31:03.962Z{"msg":"[2]R generation[0..12]: [177, 173, 173, 174, 174, 176, 176, 177, 172, 174, 174, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.132973929Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191632024-02-03T17:31:03.963Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":Wait for a query_work_queue command to finish before sending IO
191642024-02-03T17:31:03.963Z0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133070015Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191652024-02-03T17:31:03.963Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133140551Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191662024-02-03T17:31:03.963Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133169264Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191672024-02-03T17:31:03.963Z{"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.13321222Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191682024-02-03T17:31:03.963Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133236812Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191692024-02-03T17:31:03.963Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133290539Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191702024-02-03T17:31:03.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133357858Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191712024-02-03T17:31:03.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133425514Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191722024-02-03T17:31:03.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133451637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191732024-02-03T17:31:03.963Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.13350689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191742024-02-03T17:31:03.963Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133574115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191752024-02-03T17:31:03.963Z{"msg":"e3604432-ccc1-430f-819c-7bb62d943654 is now active with session: 12e53c2e-480d-47d7-ae1c-9a8181ce2548","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133608103Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191762024-02-03T17:31:03.963Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133677301Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847,"session_id":"12e53c2e-480d-47d7-ae1c-9a8181ce2548"}
191772024-02-03T17:31:03.963Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:03.133749837Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847}
191782024-02-03T17:31:03.963ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191792024-02-03T17:31:03.963ZLoading write count information from file "/tmp/test_repair_verify.data"
191802024-02-03T17:31:03.963ZRun Repair workload
191812024-02-03T17:31:03.963Z01/30 Write block 496 len 28672 data: 16 15 18 20 20 22 19
191822024-02-03T17:31:03.963Z02/30 Read block 71 len 20480
191832024-02-03T17:31:03.969Z03/30 Read block 83 len 8192
191842024-02-03T17:31:03.969Z04/30 Write block 10 len 20480 data: 22 19 20 17 15
191852024-02-03T17:31:03.969Z05/30 Read block 87 len 8192
191862024-02-03T17:31:03.969Z06/30 Read block 17 len 16384
191872024-02-03T17:31:03.969Z07/30 Write block 525 len 36864 data: 10 13 15 14 15 13 12 12 9
191882024-02-03T17:31:03.973Z08/30 Write block 404 len 36864 data: 11 16 14 15 12 14 13 13 12
191892024-02-03T17:31:03.973Z09/30 Read block 193 len 24576
191902024-02-03T17:31:03.973Z10/30 Read block 424 len 32768
191912024-02-03T17:31:03.977Z11/30 Read block 19 len 24576
191922024-02-03T17:31:03.977Z12/30 Flush
191932024-02-03T17:31:03.980Z13/30 Flush
191942024-02-03T17:31:03.980Z14/30 Write block 380 len 4096 data: 13
191952024-02-03T17:31:03.980Z15/30 Read block 179 len 28672
191962024-02-03T17:31:03.985Z16/30 Write block 297 len 4096 data: 16
191972024-02-03T17:31:03.985Z17/30 Write block 66 len 36864 data: 12 10 8 9 9 8 7 9 10
191982024-02-03T17:31:03.993Z18/30 Write block 101 len 36864 data: 15 15 18 16 19 17 20 19 20
191992024-02-03T17:31:03.993Z19/30 Read block 250 len 16384
192002024-02-03T17:31:03.993Z20/30 Read block 333 len 40960
192012024-02-03T17:31:03.993Z21/30 Flush
192022024-02-03T17:31:04.002Z22/30 Write block 505 len 24576 data: 22 25 24 21 21 16
192032024-02-03T17:31:04.002Z23/30 Write block 247 len 36864 data: 12 14 16 21 14 11 13 16 15
192042024-02-03T17:31:04.002Z24/30 Write block 565 len 24576 data: 12 14 14 11 11 12
192052024-02-03T17:31:04.002Z25/30 Read block 587 len 8192
192062024-02-03T17:31:04.002Z26/30 Flush
192072024-02-03T17:31:04.002Z27/30 Write block 139 len 28672 data: 19 16 15 13 10 11 10
192082024-02-03T17:31:04.006Z28/30 Read block 64 len 40960
192092024-02-03T17:31:04.007Z29/30 Read block 13 len 24576
192102024-02-03T17:31:04.007Z30/30 Write block 30 len 24576 data: 7 9 9 7 9 8
192112024-02-03T17:31:04.007Z----------------------------------------------------------------
192122024-02-03T17:31:04.007Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
192132024-02-03T17:31:04.007ZStates: Active Active Active
192142024-02-03T17:31:04.007ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192152024-02-03T17:31:04.007Z 1 Acked 1000 Write 7 Done Done Done false
192162024-02-03T17:31:04.007Z 2 Acked 1001 Read 5 Done Done Done false
192172024-02-03T17:31:04.007Z 3 Acked 1002 Read 2 Done Done Done false
192182024-02-03T17:31:04.007Z 4 Acked 1003 Write 5 Done Done Done false
192192024-02-03T17:31:04.007Z 5 Acked 1004 Read 2 Done Done Sent false
192202024-02-03T17:31:04.007Z 6 Acked 1005 Read 4 Done Done Sent false
192212024-02-03T17:31:04.007Z 7 Acked 1006 Write 9 Done Done Sent false
192222024-02-03T17:31:04.007Z 8 Acked 1007 Write 9 Done Done Sent false
192232024-02-03T17:31:04.007Z 9 Acked 1008 Read 6 Done Done Sent false
192242024-02-03T17:31:04.007Z 10 Acked 1009 Read 8 Done Done Sent false
192252024-02-03T17:31:04.007Z 11 Acked 1010 Read 6 Done Done Sent false
192262024-02-03T17:31:04.007Z 12 Acked 1011 Flush 0 Done Done Sent false
192272024-02-03T17:31:04.007Z 13 Acked 1012 Flush 0 Done Done Sent false
192282024-02-03T17:31:04.012Z 14 Acked 1013 Write 1 Done Done Sent false
192292024-02-03T17:31:04.012Z 15 Acked 1014 Read 7 Done Done Sent false
192302024-02-03T17:31:04.012Z 16 Acked 1015 Write 1 Done Done Sent false
192312024-02-03T17:31:04.012Z 17 Acked 1016 Write 9 Done Done Sent false
192322024-02-03T17:31:04.012Z 18 Acked 1017 Write 9 Done Done Sent false
192332024-02-03T17:31:04.012Z 19 Acked 1018 Read 4 Done Done Sent false
192342024-02-03T17:31:04.012Z 20 Acked 1019 Read 10 Done Done Sent false
192352024-02-03T17:31:04.012Z 21 Acked 1020 Flush 0 Done Done Sent false
192362024-02-03T17:31:04.012Z 22 Acked 1021 Write 6 Done Done Sent false
192372024-02-03T17:31:04.012Z 23 Acked 1022 Write 9 Done Done Sent false
192382024-02-03T17:31:04.012Z 24 Acked 1023 Write 6 Done Done Sent false
192392024-02-03T17:31:04.012Z 25 Acked 1024 Read 2 Done Done Sent false
192402024-02-03T17:31:04.012Z 26 Acked 1025 Flush 0 Done Done Sent false
192412024-02-03T17:31:04.012Z 27 Acked 1026 Write 7 Done Done Sent false
192422024-02-03T17:31:04.012Z 28 Acked 1027 Read 10 Done Done Sent false
192432024-02-03T17:31:04.012Z 29 Acked 1028 Read 6 Done Done Sent false
192442024-02-03T17:31:04.012Z 30 Acked 1029 Write 6 Sent Sent Sent false
192452024-02-03T17:31:04.013Z STATES DS:0 DS:1 DS:2 TOTAL
192462024-02-03T17:31:04.013Z New 0 0 0 0
192472024-02-03T17:31:04.013Z Sent 1 1 26 28
192482024-02-03T17:31:04.013Z Done 29 29 4 62
192492024-02-03T17:31:04.013Z Skipped 0 0 0 0
192502024-02-03T17:31:04.013Z Error 0 0 0 0
192512024-02-03T17:31:04.013ZLast Flush: 1025 1025 0
192522024-02-03T17:31:04.013ZDownstairs last five completed:
192532024-02-03T17:31:04.013ZUpstairs last five completed: 30 29 28 27 26
192542024-02-03T17:31:04.013Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:03.184319946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1847}
192552024-02-03T17:31:04.013ZWrote out file "/tmp/test_repair_verify.data"
192562024-02-03T17:31:04.017Z
192572024-02-03T17:31:04.023ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192582024-02-03T17:31:04.050ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192592024-02-03T17:31:04.050Z 0 000-019 178 178 177 324 324 323 F F T <---
192602024-02-03T17:31:04.054Z 1 020-039 173 173 173 317 317 317 T T F <---
192612024-02-03T17:31:04.054Z 2 040-059 173 173 173 317 317 317 F F F
192622024-02-03T17:31:04.054Z 3 060-079 178 178 174 326 326 318 F F F <---
192632024-02-03T17:31:04.054Z 4 080-099 174 174 174 319 319 319 F F F
192642024-02-03T17:31:04.054Z 5 100-119 178 178 176 326 326 322 F F F <---
192652024-02-03T17:31:04.054Z 6 120-139 176 176 176 322 322 322 T T F <---
192662024-02-03T17:31:04.054Z 7 140-159 177 177 177 323 323 323 T T F <---
192672024-02-03T17:31:04.055Z 8 160-179 172 172 172 313 313 313 F F F
192682024-02-03T17:31:04.055Z 9 180-199 174 174 174 319 319 319 F F F
192692024-02-03T17:31:04.055Z 10 200-219 174 174 174 319 319 319 F F F
192702024-02-03T17:31:04.055Z 11 220-239 167 167 167 307 307 307 F F F
192712024-02-03T17:31:04.055Z 12 240-259 178 178 176 327 327 321 F F F <---
192722024-02-03T17:31:04.055Z 13 260-279 176 176 176 321 321 321 F F F
192732024-02-03T17:31:04.055Z 14 280-299 178 178 176 326 326 321 F F F <---
192742024-02-03T17:31:04.055Z 15 300-319 176 176 176 321 321 321 F F F
192752024-02-03T17:31:04.055Z 16 320-339 177 177 177 323 323 323 F F F
192762024-02-03T17:31:04.055Z 17 340-359 168 168 168 308 308 308 F F F
192772024-02-03T17:31:04.055Z 18 360-379 174 174 174 319 319 319 F F F
192782024-02-03T17:31:04.055Z 19 380-399 178 178 176 326 326 322 F F F <---
192792024-02-03T17:31:04.055Z 20 400-419 178 178 174 324 324 319 F F F <---
192802024-02-03T17:31:04.055Z 21 420-439 176 176 176 322 322 322 F F F
192812024-02-03T17:31:04.055Z 22 440-459 177 177 177 323 323 323 F F F
192822024-02-03T17:31:04.055Z 23 460-479 174 174 174 319 319 319 F F F
192832024-02-03T17:31:04.055Z 24 480-499 178 178 176 324 324 321 F F T <---
192842024-02-03T17:31:04.056Z 25 500-519 178 178 176 327 327 321 F F T <---
192852024-02-03T17:31:04.056Z 26 520-539 178 178 174 324 324 319 F F F <---
192862024-02-03T17:31:04.056Z 27 540-559 176 176 176 321 321 321 F F F
192872024-02-03T17:31:04.056Z 28 560-579 178 178 176 327 327 321 F F F <---
192882024-02-03T17:31:04.056Z 29 580-599 173 173 173 317 317 317 F F F
192892024-02-03T17:31:04.056ZMax gen: 178, Max flush: 327
192902024-02-03T17:31:04.056ZError: Difference in extent metadata found!
192912024-02-03T17:31:04.059ZA Difference in extent metadata is expected here
192922024-02-03T17:31:04.060ZOn loop 89
192932024-02-03T17:31:04.060Z
192942024-02-03T17:31:04.060ZVerifying data now
192952024-02-03T17:31:10.264ZNo change after verify
192962024-02-03T17:31:10.264ZLoop: 89 Downstairs dump after verify (and repair):
192972024-02-03T17:31:10.286ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192982024-02-03T17:31:10.286Z 0 000-019 178 178 178 324 324 324 F F F
192992024-02-03T17:31:10.292Z 1 020-039 179 179 179 328 328 328 F F F
193002024-02-03T17:31:10.292Z 2 040-059 173 173 173 317 317 317 F F F
193012024-02-03T17:31:10.292Z 3 060-079 178 178 178 326 326 326 F F F
193022024-02-03T17:31:10.292Z 4 080-099 174 174 174 319 319 319 F F F
193032024-02-03T17:31:10.292Z 5 100-119 178 178 178 326 326 326 F F F
193042024-02-03T17:31:10.292Z 6 120-139 179 179 179 328 328 328 F F F
193052024-02-03T17:31:10.292Z 7 140-159 179 179 179 328 328 328 F F F
193062024-02-03T17:31:10.292Z 8 160-179 172 172 172 313 313 313 F F F
193072024-02-03T17:31:10.292Z 9 180-199 174 174 174 319 319 319 F F F
193082024-02-03T17:31:10.292Z 10 200-219 174 174 174 319 319 319 F F F
193092024-02-03T17:31:10.293Z 11 220-239 167 167 167 307 307 307 F F F
193102024-02-03T17:31:10.293Z 12 240-259 178 178 178 327 327 327 F F F
193112024-02-03T17:31:10.293Z 13 260-279 176 176 176 321 321 321 F F F
193122024-02-03T17:31:10.293Z 14 280-299 178 178 178 326 326 326 F F F
193132024-02-03T17:31:10.293Z 15 300-319 176 176 176 321 321 321 F F F
193142024-02-03T17:31:10.293Z 16 320-339 177 177 177 323 323 323 F F F
193152024-02-03T17:31:10.293Z 17 340-359 168 168 168 308 308 308 F F F
193162024-02-03T17:31:10.293Z 18 360-379 174 174 174 319 319 319 F F F
193172024-02-03T17:31:10.293Z 19 380-399 178 178 178 326 326 326 F F F
193182024-02-03T17:31:10.293Z 20 400-419 178 178 178 324 324 324 F F F
193192024-02-03T17:31:10.293Z 21 420-439 176 176 176 322 322 322 F F F
193202024-02-03T17:31:10.293Z 22 440-459 177 177 177 323 323 323 F F F
193212024-02-03T17:31:10.293Z 23 460-479 174 174 174 319 319 319 F F F
193222024-02-03T17:31:10.293Z 24 480-499 178 178 178 324 324 324 F F F
193232024-02-03T17:31:10.293Z 25 500-519 178 178 178 327 327 327 F F F
193242024-02-03T17:31:10.294Z 26 520-539 178 178 178 324 324 324 F F F
193252024-02-03T17:31:10.294Z 27 540-559 176 176 176 321 321 321 F F F
193262024-02-03T17:31:10.294Z 28 560-579 178 178 178 327 327 327 F F F
193272024-02-03T17:31:10.294Z 29 580-599 173 173 173 317 317 317 F F F
193282024-02-03T17:31:10.294ZMax gen: 179, Max flush: 328
193292024-02-03T17:31:10.297Z
193302024-02-03T17:31:10.298ZBegin loop 90
193312024-02-03T17:31:10.298ZDownstairs to restart: 2
193322024-02-03T17:31:10.324Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.501067321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855}
193332024-02-03T17:31:10.324Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.50160836Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855}
193342024-02-03T17:31:10.331Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.501690687Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855}
193352024-02-03T17:31:10.331ZCrucible runtime is spawned
193362024-02-03T17:31:10.331Z{"msg":"Crucible f79c8846-3bae-4dd2-9cb3-aba254235ae7 has session id: 58b82425-8e7f-4d8a-8bb4-788e3901dcc6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.501985732Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193372024-02-03T17:31:10.331Z{"msg":"Upstairs opts: Upstairs UUID: f79c8846-3bae-4dd2-9cb3-aba254235ae7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502084714Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193382024-02-03T17:31:10.331Z{"msg":"Crucible stats registered with UUID: f79c8846-3bae-4dd2-9cb3-aba254235ae7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502155624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193392024-02-03T17:31:10.331Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502198734Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855}
193402024-02-03T17:31:10.331Z{"msg":"f79c8846-3bae-4dd2-9cb3-aba254235ae7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502307628Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193412024-02-03T17:31:10.331Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502445821Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193422024-02-03T17:31:10.331Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502587849Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193432024-02-03T17:31:10.331Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502619727Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193442024-02-03T17:31:10.331Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502663206Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193452024-02-03T17:31:10.331Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502705289Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193462024-02-03T17:31:10.332Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502749024Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193472024-02-03T17:31:10.332Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502858666Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193482024-02-03T17:31:10.332Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.502962964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193492024-02-03T17:31:10.332Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.503080057Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193502024-02-03T17:31:10.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.503135933Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193512024-02-03T17:31:10.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.503274373Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193522024-02-03T17:31:10.332Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.503348788Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193532024-02-03T17:31:10.332Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.503883148Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193542024-02-03T17:31:10.332Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.503956301Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193552024-02-03T17:31:10.332Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.504093056Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193562024-02-03T17:31:10.332Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.50420189Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193572024-02-03T17:31:10.332Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.505097085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193582024-02-03T17:31:10.332Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.505208451Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193592024-02-03T17:31:10.332Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.506045087Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193602024-02-03T17:31:10.332Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.506103387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193612024-02-03T17:31:10.332Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.506127946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193622024-02-03T17:31:10.332Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.506243499Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193632024-02-03T17:31:10.332Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.506304263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193642024-02-03T17:31:10.332Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.506330046Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193652024-02-03T17:31:10.333Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.506407407Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193662024-02-03T17:31:10.333Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.506474043Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193672024-02-03T17:31:10.333Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.507263975Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193682024-02-03T17:31:10.337Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.507983396Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193692024-02-03T17:31:10.337Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508056444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193702024-02-03T17:31:10.337Z{"msg":"[0]R flush_numbers[0..12]: [324, 328, 317, 326, 319, 326, 328, 328, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508102602Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193712024-02-03T17:31:10.337Z{"msg":"[0]R generation[0..12]: [178, 179, 173, 178, 174, 178, 179, 179, 172, 174, 174, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508148602Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193722024-02-03T17:31:10.337Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508174845Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193732024-02-03T17:31:10.337Z{"msg":"[1]R flush_numbers[0..12]: [324, 328, 317, 326, 319, 326, 328, 328, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508216822Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193742024-02-03T17:31:10.337Z{"msg":"[1]R generation[0..12]: [178, 179, 173, 178, 174, 178, 179, 179, 172, 174, 174, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508241539Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193752024-02-03T17:31:10.337Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508284287Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
193762024-02-03T17:31:10.337Zsession_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193772024-02-03T17:31:10.337Z{"msg":"[2]R flush_numbers[0..12]: [324, 328, 317, 326, 319, 326, 328, 328, 313, 319, 319, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508314312Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193782024-02-03T17:31:10.337Z{"msg":"[2]R generation[0..12]: [178, 179, 173, 178, 174, 178, 179, 179, 172, 174, 174, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508370029Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193792024-02-03T17:31:10.337Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508395037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193802024-02-03T17:31:10.337Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508441325Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193812024-02-03T17:31:10.338Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.50846492Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193822024-02-03T17:31:10.338Z{"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.50850883Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193832024-02-03T17:31:10.338Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508550817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193842024-02-03T17:31:10.338Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508574687Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193852024-02-03T17:31:10.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508621109Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193862024-02-03T17:31:10.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508646792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193872024-02-03T17:31:10.338Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508694637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193882024-02-03T17:31:10.338Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508719945Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193892024-02-03T17:31:10.338Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508764831Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193902024-02-03T17:31:10.338Z{"msg":"f79c8846-3bae-4dd2-9cb3-aba254235ae7 is now active with session: 58b82425-8e7f-4d8a-8bb4-788e3901dcc6","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.50878792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193912024-02-03T17:31:10.338Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.50883223Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855,"session_id":"58b82425-8e7f-4d8a-8bb4-788e3901dcc6"}
193922024-02-03T17:31:10.338Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:09.508855845Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855}
193932024-02-03T17:31:10.338ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193942024-02-03T17:31:10.338ZLoading write count information from file "/tmp/test_repair_verify.data"
193952024-02-03T17:31:10.338ZRun Repair workload
193962024-02-03T17:31:10.338Z01/30 Write block 358 len 16384 data: 18 18 17 18
193972024-02-03T17:31:10.338Z02/30 Write block 300 len 16384 data: 11 10 10 10
193982024-02-03T17:31:10.338Z03/30 Read block 135 len 20480
193992024-02-03T17:31:10.345Z04/30 Write block 38 len 40960 data: 13 14 12 10 12 11 10 8 7 7
194002024-02-03T17:31:10.345Z05/30 Write block 461 len 12288 data: 18 20 18
194012024-02-03T17:31:10.345Z06/30 Flush
194022024-02-03T17:31:10.349Z07/30 Write block 188 len 36864 data: 11 12 11 11 10 8 7 9 11
194032024-02-03T17:31:10.349Z08/30 Write block 369 len 12288 data: 12 12 14
194042024-02-03T17:31:10.349Z09/30 Write block 27 len 20480 data: 12 13 10 8 10
194052024-02-03T17:31:10.353Z10/30 Write block 290 len 20480 data: 10 11 11 13 15
194062024-02-03T17:31:10.353Z11/30 Write block 110 len 8192 data: 19 16
194072024-02-03T17:31:10.353Z12/30 Write block 308 len 4096 data: 15
194082024-02-03T17:31:10.357Z13/30 Write block 296 len 36864 data: 15 17 12 9 12 11 11 11 11
194092024-02-03T17:31:10.357Z14/30 Flush
194102024-02-03T17:31:10.363Z15/30 Write block 210 len 24576 data: 13 11 10 10 11 13
194112024-02-03T17:31:10.363Z16/30 Read block 131 len 8192
194122024-02-03T17:31:10.367Z17/30 Write block 145 len 32768 data: 11 7 6 7 8 13 14 10
194132024-02-03T17:31:10.367Z18/30 Read block 504 len 4096
194142024-02-03T17:31:10.367Z19/30 Flush
194152024-02-03T17:31:10.367Z20/30 Flush
194162024-02-03T17:31:10.371Z21/30 Write block 93 len 32768 data: 12 12 12 13 13 11 13 15
194172024-02-03T17:31:10.371Z22/30 Read block 541 len 20480
194182024-02-03T17:31:10.380Z23/30 Read block 417 len 12288
194192024-02-03T17:31:10.380Z24/30 Write block 297 len 40960 data: 18 13 10 13 12 12 12 12 14 16
194202024-02-03T17:31:10.381Z25/30 Read block 278 len 32768
194212024-02-03T17:31:10.384Z26/30 Write block 211 len 36864 data: 12 11 11 12 14 11 11 10 12
194222024-02-03T17:31:10.385Z27/30 Write block 218 len 8192 data: 11 13
194232024-02-03T17:31:10.385Z28/30 Write block 101 len 24576 data: 16 16 19 17 20 18
194242024-02-03T17:31:10.385Z29/30 Read block 283 len 20480
194252024-02-03T17:31:10.385Z30/30 Write block 461 len 36864 data: 19 21 19 18 17 14 14 13 13
194262024-02-03T17:31:10.385Z----------------------------------------------------------------
194272024-02-03T17:31:10.385Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:16
194282024-02-03T17:31:10.385ZStates: Active Active Active
194292024-02-03T17:31:10.385ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194302024-02-03T17:31:10.385Z 15 Acked 1014 Write 6 Done Done Sent false
194312024-02-03T17:31:10.385Z 16 Acked 1015 Read 2 Done Done Sent false
194322024-02-03T17:31:10.385Z 17 Acked 1016 Write 8 Done Done Sent false
194332024-02-03T17:31:10.385Z 18 Acked 1017 Read 1 Done Done Sent false
194342024-02-03T17:31:10.385Z 19 Acked 1018 Flush 0 Done Done Sent false
194352024-02-03T17:31:10.385Z 20 Acked 1019 Flush 0 Done Done Sent false
194362024-02-03T17:31:10.385Z 21 Acked 1020 Write 8 Done Done Sent false
194372024-02-03T17:31:10.385Z 22 Acked 1021 Read 5 Done Done Sent false
194382024-02-03T17:31:10.385Z 23 Acked 1022 Read 3 Done Done Sent false
194392024-02-03T17:31:10.385Z 24 Acked 1023 Write 10 Done Done Sent false
194402024-02-03T17:31:10.385Z 25 Acked 1024 Read 8 Done Done Sent false
194412024-02-03T17:31:10.385Z 26 Acked 1025 Write 9 Done Done Sent false
194422024-02-03T17:31:10.386Z 27 Acked 1026 Write 2 Done Done Sent false
194432024-02-03T17:31:10.386Z 28 Acked 1027 Write 6 Done Done Sent false
194442024-02-03T17:31:10.386Z 29 Acked 1028 Read 5 Done Done Sent false
194452024-02-03T17:31:10.386Z 30 Acked 1029 Write 9 Sent Sent Sent false
194462024-02-03T17:31:10.386Z STATES DS:0 DS:1 DS:2 TOTAL
194472024-02-03T17:31:10.386Z New 0 0 0 0
194482024-02-03T17:31:10.386Z Sent 1 1 16 18
194492024-02-03T17:31:10.386Z Done 15 15 0 30
194502024-02-03T17:31:10.386Z Skipped 0 0 0 0
194512024-02-03T17:31:10.386Z Error 0 0 0 0
194522024-02-03T17:31:10.386ZLast Flush: 1019 1019 1013
194532024-02-03T17:31:10.386ZDownstairs last five completed: 1013 1012 1011 1010 1009
194542024-02-03T17:31:10.386ZUpstairs last five completed: 30 29 28 27 26
194552024-02-03T17:31:10.386Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:09.561685943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1855}
194562024-02-03T17:31:10.400ZWrote out file "/tmp/test_repair_verify.data"
194572024-02-03T17:31:10.404Z
194582024-02-03T17:31:10.404ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194592024-02-03T17:31:10.428ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194602024-02-03T17:31:10.428Z 0 000-019 178 178 178 324 324 324 F F F
194612024-02-03T17:31:10.428Z 1 020-039 180 180 180 330 330 330 F F F
194622024-02-03T17:31:10.428Z 2 040-059 180 180 180 329 329 329 F F F
194632024-02-03T17:31:10.428Z 3 060-079 178 178 178 326 326 326 F F F
194642024-02-03T17:31:10.432Z 4 080-099 174 174 174 319 319 319 T T F <---
194652024-02-03T17:31:10.432Z 5 100-119 180 180 180 330 330 330 T T F <---
194662024-02-03T17:31:10.432Z 6 120-139 179 179 179 328 328 328 F F F
194672024-02-03T17:31:10.432Z 7 140-159 180 180 179 331 331 328 F F F <---
194682024-02-03T17:31:10.432Z 8 160-179 172 172 172 313 313 313 F F F
194692024-02-03T17:31:10.432Z 9 180-199 180 180 180 330 330 330 F F F
194702024-02-03T17:31:10.432Z 10 200-219 180 180 174 331 331 319 T T F <---
194712024-02-03T17:31:10.433Z 11 220-239 167 167 167 307 307 307 F F F
194722024-02-03T17:31:10.433Z 12 240-259 178 178 178 327 327 327 F F F
194732024-02-03T17:31:10.433Z 13 260-279 176 176 176 321 321 321 F F F
194742024-02-03T17:31:10.433Z 14 280-299 180 180 180 330 330 330 T T F <---
194752024-02-03T17:31:10.433Z 15 300-319 180 180 180 330 330 330 T T F <---
194762024-02-03T17:31:10.433Z 16 320-339 177 177 177 323 323 323 F F F
194772024-02-03T17:31:10.433Z 17 340-359 180 180 180 329 329 329 F F F
194782024-02-03T17:31:10.433Z 18 360-379 180 180 180 330 330 330 F F F
194792024-02-03T17:31:10.433Z 19 380-399 178 178 178 326 326 326 F F F
194802024-02-03T17:31:10.433Z 20 400-419 178 178 178 324 324 324 F F F
194812024-02-03T17:31:10.433Z 21 420-439 176 176 176 322 322 322 F F F
194822024-02-03T17:31:10.433Z 22 440-459 177 177 177 323 323 323 F F F
194832024-02-03T17:31:10.433Z 23 460-479 180 180 180 329 329 329 T T F <---
194842024-02-03T17:31:10.434Z 24 480-499 178 178 178 324 324 324 F F F
194852024-02-03T17:31:10.434Z 25 500-519 178 178 178 327 327 327 F F F
194862024-02-03T17:31:10.434Z 26 520-539 178 178 178 324 324 324 F F F
194872024-02-03T17:31:10.434Z 27 540-559 176 176 176 321 321 321 F F F
194882024-02-03T17:31:10.434Z 28 560-579 178 178 178 327 327 327 F F F
194892024-02-03T17:31:10.434Z 29 580-599 173 173 173 317 317 317 F F F
194902024-02-03T17:31:10.434ZMax gen: 180, Max flush: 331
194912024-02-03T17:31:10.434ZError: Difference in extent metadata found!
194922024-02-03T17:31:10.438ZA Difference in extent metadata is expected here
194932024-02-03T17:31:10.438ZOn loop 90
194942024-02-03T17:31:10.438Z
194952024-02-03T17:31:10.438ZVerifying data now
194962024-02-03T17:31:15.150ZNo change after verify
194972024-02-03T17:31:15.150ZLoop: 90 Downstairs dump after verify (and repair):
194982024-02-03T17:31:15.175ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194992024-02-03T17:31:15.175Z 0 000-019 178 178 178 324 324 324 F F F
195002024-02-03T17:31:15.179Z 1 020-039 180 180 180 330 330 330 F F F
195012024-02-03T17:31:15.179Z 2 040-059 180 180 180 329 329 329 F F F
195022024-02-03T17:31:15.179Z 3 060-079 178 178 178 326 326 326 F F F
195032024-02-03T17:31:15.179Z 4 080-099 181 181 181 332 332 332 F F F
195042024-02-03T17:31:15.179Z 5 100-119 181 181 181 332 332 332 F F F
195052024-02-03T17:31:15.179Z 6 120-139 179 179 179 328 328 328 F F F
195062024-02-03T17:31:15.179Z 7 140-159 180 180 180 331 331 331 F F F
195072024-02-03T17:31:15.179Z 8 160-179 172 172 172 313 313 313 F F F
195082024-02-03T17:31:15.179Z 9 180-199 180 180 180 330 330 330 F F F
195092024-02-03T17:31:15.179Z 10 200-219 181 181 181 332 332 332 F F F
195102024-02-03T17:31:15.179Z 11 220-239 167 167 167 307 307 307 F F F
195112024-02-03T17:31:15.179Z 12 240-259 178 178 178 327 327 327 F F F
195122024-02-03T17:31:15.179Z 13 260-279 176 176 176 321 321 321 F F F
195132024-02-03T17:31:15.179Z 14 280-299 181 181 181 332 332 332 F F F
195142024-02-03T17:31:15.179Z 15 300-319 181 181 181 332 332 332 F F F
195152024-02-03T17:31:15.180Z 16 320-339 177 177 177 323 323 323 F F F
195162024-02-03T17:31:15.180Z 17 340-359 180 180 180 329 329 329 F F F
195172024-02-03T17:31:15.180Z 18 360-379 180 180 180 330 330 330 F F F
195182024-02-03T17:31:15.180Z 19 380-399 178 178 178 326 326 326 F F F
195192024-02-03T17:31:15.180Z 20 400-419 178 178 178 324 324 324 F F F
195202024-02-03T17:31:15.180Z 21 420-439 176 176 176 322 322 322 F F F
195212024-02-03T17:31:15.180Z 22 440-459 177 177 177 323 323 323 F F F
195222024-02-03T17:31:15.180Z 23 460-479 181 181 181 332 332 332 F F F
195232024-02-03T17:31:15.180Z 24 480-499 178 178 178 324 324 324 F F F
195242024-02-03T17:31:15.180Z 25 500-519 178 178 178 327 327 327 F F F
195252024-02-03T17:31:15.180Z 26 520-539 178 178 178 324 324 324 F F F
195262024-02-03T17:31:15.180Z 27 540-559 176 176 176 321 321 321 F F F
195272024-02-03T17:31:15.180Z 28 560-579 178 178 178 327 327 327 F F F
195282024-02-03T17:31:15.180Z 29 580-599 173 173 173 317 317 317 F F F
195292024-02-03T17:31:15.180ZMax gen: 181, Max flush: 332
195302024-02-03T17:31:15.184Z
195312024-02-03T17:31:15.184ZBegin loop 91
195322024-02-03T17:31:15.184ZDownstairs to restart: 0
195332024-02-03T17:31:15.221Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.393412635Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863}
195342024-02-03T17:31:15.221Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","Crucible runtime is spawned
195352024-02-03T17:31:15.243Zlevel":30,"time":"2024-02-03T17:31:14.394037499Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863}
195362024-02-03T17:31:15.243Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.39412664Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863}
195372024-02-03T17:31:15.243Z{"msg":"Crucible 8c866fed-a0e1-4df8-a6e1-86904262a3fc has session id: 560d9ee8-8932-4d0c-a42b-827235c462db","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394194682Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195382024-02-03T17:31:15.243Z{"msg":"Upstairs opts: Upstairs UUID: 8c866fed-a0e1-4df8-a6e1-86904262a3fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394256429Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195392024-02-03T17:31:15.243Z{"msg":"Crucible stats registered with UUID: 8c866fed-a0e1-4df8-a6e1-86904262a3fc","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394297193Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195402024-02-03T17:31:15.243Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394331627Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863}
195412024-02-03T17:31:15.243Z{"msg":"8c866fed-a0e1-4df8-a6e1-86904262a3fc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394375759Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195422024-02-03T17:31:15.243Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394398923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195432024-02-03T17:31:15.243Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394422432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195442024-02-03T17:31:15.243Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394478225Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195452024-02-03T17:31:15.243Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394503093Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195462024-02-03T17:31:15.243Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394525442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195472024-02-03T17:31:15.243Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394547594Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195482024-02-03T17:31:15.243Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394669991Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195492024-02-03T17:31:15.243Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394797043Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195502024-02-03T17:31:15.243Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394888566Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195512024-02-03T17:31:15.243Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.394990406Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195522024-02-03T17:31:15.243Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:14.395042121Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195532024-02-03T17:31:15.243Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:14.395142512Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195542024-02-03T17:31:15.244Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.395189096Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195552024-02-03T17:31:15.244Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:14.395234738Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195562024-02-03T17:31:15.244Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.395273368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195572024-02-03T17:31:15.244Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.395311247Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195582024-02-03T17:31:15.244Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.395524586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195592024-02-03T17:31:15.244Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.396361495Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195602024-02-03T17:31:15.244Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.396423171Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195612024-02-03T17:31:15.244Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.396688161Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195622024-02-03T17:31:15.244Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.396801855Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195632024-02-03T17:31:15.244Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.397342424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195642024-02-03T17:31:15.244Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.397601639Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195652024-02-03T17:31:15.244Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.398250203Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195662024-02-03T17:31:15.244Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.398333577Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195672024-02-03T17:31:15.244Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.398360547Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195682024-02-03T17:31:15.244Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.398452074Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195692024-02-03T17:31:15.244Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.398506222Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195702024-02-03T17:31:15.244Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:14.398532488Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195712024-02-03T17:31:16.219Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.396330479Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195722024-02-03T17:31:16.219Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.396461766Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195732024-02-03T17:31:16.224Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.397698256Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195742024-02-03T17:31:16.224Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.397764655Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195752024-02-03T17:31:16.224Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.398494405Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195762024-02-03T17:31:16.224Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399350147Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195772024-02-03T17:31:16.224Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.39943369Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195782024-02-03T17:31:16.224Z{"msg":"[0]R flush_numbers[0..12]: [324, 330, 329, 326, 332, 332, 328, 331, 313, 330, 332, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399469299Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195792024-02-03T17:31:16.224Z{"msg":"[0]R generation[0..12]: [178, 180, 180, 178, 181, 181, 179, 180, 172, 180, 181, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.39950375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195802024-02-03T17:31:16.224Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399549654Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195812024-02-03T17:31:16.224Z{"msg":"[1]R flush_numbers[0..12]: [324, 330, 329, 326, 332, 332, 328, 331, 313, 330, 332, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399575586Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195822024-02-03T17:31:16.225Z{"msg":"[1]R generation[0..12]: [178, 180, 180, 178, 181, 181, 179, 180, 172, 180, 181, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399617765Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195832024-02-03T17:31:16.225Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399643342Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195842024-02-03T17:31:16.225Z{"msg":"[2]R flush_numbers[0..12]: [324, 330, 329, 326, 332, 332, 328, 331, 313, 330, 332, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399689313Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195852024-02-03T17:31:16.225Z{"msg":"[2]R generation[0..12]: [178, 180, 180, 178, 181, 181, 179, 180, 172, 180, 181, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399714226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195862024-02-03T17:31:16.225ZWait for a query_work_queue command to finish before sending IO
195872024-02-03T17:31:16.225Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399810388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195882024-02-03T17:31:16.225Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399860556Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195892024-02-03T17:31:16.225Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399885674Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195902024-02-03T17:31:16.225Z{"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399908684Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195912024-02-03T17:31:16.225Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399949671Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195922024-02-03T17:31:16.225Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399973604Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195932024-02-03T17:31:16.225Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.399994992Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195942024-02-03T17:31:16.225Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.40001775Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195952024-02-03T17:31:16.225Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.400039362Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195962024-02-03T17:31:16.225Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.400080946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195972024-02-03T17:31:16.225Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.400130128Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195982024-02-03T17:31:16.225Z{"msg":"8c866fed-a0e1-4df8-a6e1-86904262a3fc is now active with session: 560d9ee8-8932-4d0c-a42b-827235c462db","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.400170424Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
195992024-02-03T17:31:16.225Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.400239037Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863,"session_id":"560d9ee8-8932-4d0c-a42b-827235c462db"}
196002024-02-03T17:31:16.225Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:15.400277755Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863}
196012024-02-03T17:31:16.225ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
196022024-02-03T17:31:16.229ZLoading write count information from file "/tmp/test_repair_verify.data"
196032024-02-03T17:31:16.229ZRun Repair workload
196042024-02-03T17:31:16.229Z01/30 Write block 309 len 12288 data: 15 13 12
196052024-02-03T17:31:16.229Z02/30 Read block 370 len 40960
196062024-02-03T17:31:16.229Z03/30 Read block 446 len 4096
196072024-02-03T17:31:16.232Z04/30 Write block 15 len 32768 data: 14 13 11 10 9 6 9 11
196082024-02-03T17:31:16.232Z05/30 Read block 392 len 12288
196092024-02-03T17:31:16.235Z06/30 Write block 471 len 32768 data: 14 12 11 13 12 12 12 13
196102024-02-03T17:31:16.235Z07/30 Flush
196112024-02-03T17:31:16.238Z08/30 Flush
196122024-02-03T17:31:16.241Z09/30 Write block 579 len 12288 data: 21 21 22
196132024-02-03T17:31:16.241Z10/30 Read block 321 len 20480
196142024-02-03T17:31:16.241Z11/30 Write block 3 len 40960 data: 12 14 16 16 17 17 18 23 20 21
196152024-02-03T17:31:16.244Z12/30 Write block 544 len 20480 data: 13 14 12 13 19
196162024-02-03T17:31:16.244Z13/30 Write block 561 len 32768 data: 12 9 10 10 13 15 15 12
196172024-02-03T17:31:16.247Z14/30 Write block 2 len 24576 data: 11 13 15 17 17 18
196182024-02-03T17:31:16.248Z15/30 Write block 195 len 4096 data: 10
196192024-02-03T17:31:16.248Z16/30 Write block 284 len 16384 data: 16 13 12 13
196202024-02-03T17:31:16.251Z17/30 Read block 461 len 4096
196212024-02-03T17:31:16.251Z18/30 Write block 149 len 36864 data: 9 14 15 11 11 13 16 16 16
196222024-02-03T17:31:16.251Z19/30 Read block 544 len 36864
196232024-02-03T17:31:16.254Z20/30 Write block 408 len 32768 data: 13 15 14 14 13 15 12 8
196242024-02-03T17:31:16.257Z21/30 Write block 519 len 32768 data: 11 9 9 8 7 7 11 14
196252024-02-03T17:31:16.257Z22/30 Write block 532 len 16384 data: 13 10 8 10
196262024-02-03T17:31:16.260Z23/30 Write block 164 len 32768 data: 12 10 9 10 9 14 14 14
196272024-02-03T17:31:16.260Z24/30 Read block 353 len 40960
196282024-02-03T17:31:16.263Z25/30 Write block 391 len 24576 data: 15 11 11 12 14 15
196292024-02-03T17:31:16.264Z26/30 Write block 485 len 40960 data: 10 12 12 13 15 16 15 13 15 14
196302024-02-03T17:31:16.264Z27/30 Read block 151 len 36864
196312024-02-03T17:31:16.270Z28/30 Write block 76 len 28672 data: 12 12 11 10 11 10 13
196322024-02-03T17:31:16.270Z29/30 Read block 270 len 28672
196332024-02-03T17:31:16.275Z30/30 Read block 293 len 32768
196342024-02-03T17:31:16.275Z----------------------------------------------------------------
196352024-02-03T17:31:16.276Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196362024-02-03T17:31:16.276ZStates: Active Active Active
196372024-02-03T17:31:16.276ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196382024-02-03T17:31:16.276Z 1 Acked 1000 Write 3 Done Done Done false
196392024-02-03T17:31:16.276Z 2 Acked 1001 Read 10 Sent Done Done false
196402024-02-03T17:31:16.276Z 3 Acked 1002 Read 1 Sent Done Done false
196412024-02-03T17:31:16.276Z 4 Acked 1003 Write 8 Sent Done Done false
196422024-02-03T17:31:16.276Z 5 Acked 1004 Read 3 Sent Done Done false
196432024-02-03T17:31:16.276Z 6 Acked 1005 Write 8 Sent Done Done false
196442024-02-03T17:31:16.276Z 7 Acked 1006 Flush 0 Sent Done Done false
196452024-02-03T17:31:16.276Z 8 Acked 1007 Flush 0 Sent Done Done false
196462024-02-03T17:31:16.276Z 9 Acked 1008 Write 3 Sent Done Done false
196472024-02-03T17:31:16.276Z 10 Acked 1009 Read 5 Sent Done Done false
196482024-02-03T17:31:16.276Z 11 Acked 1010 Write 10 Sent Done Done false
196492024-02-03T17:31:16.276Z 12 Acked 1011 Write 5 Sent Done Done false
196502024-02-03T17:31:16.276Z 13 Acked 1012 Write 8 Sent Done Done false
196512024-02-03T17:31:16.276Z 14 Acked 1013 Write 6 Sent Done Done false
196522024-02-03T17:31:16.276Z 15 Acked 1014 Write 1 Sent Done Done false
196532024-02-03T17:31:16.276Z 16 Acked 1015 Write 4 Sent Done Done false
196542024-02-03T17:31:16.276Z 17 Acked 1016 Read 1 Sent Done Done false
196552024-02-03T17:31:16.276Z 18 Acked 1017 Write 9 Sent Done Done false
196562024-02-03T17:31:16.276Z 19 Acked 1018 Read 9 Sent Done Done false
196572024-02-03T17:31:16.276Z 20 Acked 1019 Write 8 Sent Done Done false
196582024-02-03T17:31:16.276Z 21 Acked 1020 Write 8 Sent Done Done false
196592024-02-03T17:31:16.276Z 22 Acked 1021 Write 4 Sent Done Done false
196602024-02-03T17:31:16.277Z 23 Acked 1022 Write 8 Sent Done Done false
196612024-02-03T17:31:16.277Z 24 Acked 1023 Read 10 Sent Done Done false
196622024-02-03T17:31:16.277Z 25 Acked 1024 Write 6 Sent Done Done false
196632024-02-03T17:31:16.277Z 26 Acked 1025 Write 10 Sent Done Done false
196642024-02-03T17:31:16.277Z 27 Acked 1026 Read 9 Sent Done Done false
196652024-02-03T17:31:16.277Z 28 Acked 1027 Write 7 Sent Done Done false
196662024-02-03T17:31:16.277Z 29 Acked 1028 Read 7 Sent Done Done false
196672024-02-03T17:31:16.277Z 30 Acked 1029 Read 8 Sent Sent Done false
196682024-02-03T17:31:16.277Z STATES DS:0 DS:1 DS:2 TOTAL
196692024-02-03T17:31:16.277Z New 0 0 0 0
196702024-02-03T17:31:16.277Z Sent 29 1 0 30
196712024-02-03T17:31:16.277Z Done 1 29 30 60
196722024-02-03T17:31:16.277Z Skipped 0 0 0 0
196732024-02-03T17:31:16.277Z Error 0 0 0 0
196742024-02-03T17:31:16.277ZLast Flush: 0 1007 1007
196752024-02-03T17:31:16.277ZDownstairs last five completed:
196762024-02-03T17:31:16.277ZUpstairs last five completed: 30 29 28 27 26
196772024-02-03T17:31:16.277Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:15.450153817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1863}
196782024-02-03T17:31:16.277ZWrote out file "/tmp/test_repair_verify.data"
196792024-02-03T17:31:16.281Z
196802024-02-03T17:31:16.288ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196812024-02-03T17:31:17.549ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196822024-02-03T17:31:17.549Z 0 000-019 178 182 182 324 333 333 F T T <---
196832024-02-03T17:31:17.549Z 1 020-039 180 182 182 330 333 333 F F F <---
196842024-02-03T17:31:17.549Z 2 040-059 180 180 180 329 329 329 F F F
196852024-02-03T17:31:17.549Z 3 060-079 178 178 178 326 326 326 F T T <---
196862024-02-03T17:31:17.549Z 4 080-099 181 181 181 332 332 332 F T T <---
196872024-02-03T17:31:17.549Z 5 100-119 181 181 181 332 332 332 F F F
196882024-02-03T17:31:17.549Z 6 120-139 179 179 179 328 328 328 F F F
196892024-02-03T17:31:17.549Z 7 140-159 180 180 180 331 331 331 F T T <---
196902024-02-03T17:31:17.549Z 8 160-179 172 172 172 313 313 313 F T T <---
196912024-02-03T17:31:17.560Z 9 180-199 180 180 180 330 330 330 F T T <---
196922024-02-03T17:31:17.560Z 10 200-219 181 181 181 332 332 332 F F F
196932024-02-03T17:31:17.560Z 11 220-239 167 167 167 307 307 307 F F F
196942024-02-03T17:31:17.561Z 12 240-259 178 178 178 327 327 327 F F F
196952024-02-03T17:31:17.561Z 13 260-279 176 176 176 321 321 321 F F F
196962024-02-03T17:31:17.561Z 14 280-299 181 181 181 332 332 332 F T T <---
196972024-02-03T17:31:17.561Z 15 300-319 181 182 182 332 333 333 T F F <---
196982024-02-03T17:31:17.561Z 16 320-339 177 177 177 323 323 323 F F F
196992024-02-03T17:31:17.561Z 17 340-359 180 180 180 329 329 329 F F F
197002024-02-03T17:31:17.561Z 18 360-379 180 180 180 330 330 330 F F F
197012024-02-03T17:31:17.561Z 19 380-399 178 178 178 326 326 326 F T T <---
197022024-02-03T17:31:17.561Z 20 400-419 178 178 178 324 324 324 F T T <---
197032024-02-03T17:31:17.561Z 21 420-439 176 176 176 322 322 322 F F F
197042024-02-03T17:31:17.561Z 22 440-459 177 177 177 323 323 323 F F F
197052024-02-03T17:31:17.561Z 23 460-479 181 182 182 332 333 333 F F F <---
197062024-02-03T17:31:17.561Z 24 480-499 178 178 178 324 324 324 F T T <---
197072024-02-03T17:31:17.561Z 25 500-519 178 178 178 327 327 327 F T T <---
197082024-02-03T17:31:17.561Z 26 520-539 178 178 178 324 324 324 F T T <---
197092024-02-03T17:31:17.562Z 27 540-559 176 176 176 321 321 321 F T T <---
197102024-02-03T17:31:17.562Z 28 560-579 178 178 178 327 327 327 F T T <---
197112024-02-03T17:31:17.562Z 29 580-599 173 173 173 317 317 317 F T T <---
197122024-02-03T17:31:17.562ZMax gen: 182, Max flush: 333
197132024-02-03T17:31:17.562ZError: Difference in extent metadata found!
197142024-02-03T17:31:17.562ZA Difference in extent metadata is expected here
197152024-02-03T17:31:17.562ZOn loop 91
197162024-02-03T17:31:17.562Z
197172024-02-03T17:31:17.562ZVerifying data now
197182024-02-03T17:31:21.874ZNo change after verify
197192024-02-03T17:31:21.875ZLoop: 91 Downstairs dump after verify (and repair):
197202024-02-03T17:31:21.897ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197212024-02-03T17:31:21.897Z 0 000-019 183 183 183 334 334 334 F F F
197222024-02-03T17:31:21.897Z 1 020-039 182 182 182 333 333 333 F F F
197232024-02-03T17:31:21.897Z 2 040-059 180 180 180 329 329 329 F F F
197242024-02-03T17:31:21.898Z 3 060-079 183 183 183 334 334 334 F F F
197252024-02-03T17:31:21.898Z 4 080-099 183 183 183 334 334 334 F F F
197262024-02-03T17:31:21.898Z 5 100-119 181 181 181 332 332 332 F F F
197272024-02-03T17:31:21.907Z 6 120-139 179 179 179 328 328 328 F F F
197282024-02-03T17:31:21.907Z 7 140-159 183 183 183 334 334 334 F F F
197292024-02-03T17:31:21.907Z 8 160-179 183 183 183 334 334 334 F F F
197302024-02-03T17:31:21.908Z 9 180-199 183 183 183 334 334 334 F F F
197312024-02-03T17:31:21.908Z 10 200-219 181 181 181 332 332 332 F F F
197322024-02-03T17:31:21.908Z 11 220-239 167 167 167 307 307 307 F F F
197332024-02-03T17:31:21.908Z 12 240-259 178 178 178 327 327 327 F F F
197342024-02-03T17:31:21.908Z 13 260-279 176 176 176 321 321 321 F F F
197352024-02-03T17:31:21.908Z 14 280-299 183 183 183 334 334 334 F F F
197362024-02-03T17:31:21.908Z 15 300-319 182 182 182 333 333 333 F F F
197372024-02-03T17:31:21.908Z 16 320-339 177 177 177 323 323 323 F F F
197382024-02-03T17:31:21.908Z 17 340-359 180 180 180 329 329 329 F F F
197392024-02-03T17:31:21.908Z 18 360-379 180 180 180 330 330 330 F F F
197402024-02-03T17:31:21.908Z 19 380-399 183 183 183 334 334 334 F F F
197412024-02-03T17:31:21.908Z 20 400-419 183 183 183 334 334 334 F F F
197422024-02-03T17:31:21.909Z 21 420-439 176 176 176 322 322 322 F F F
197432024-02-03T17:31:21.909Z 22 440-459 177 177 177 323 323 323 F F F
197442024-02-03T17:31:21.909Z 23 460-479 182 182 182 333 333 333 F F F
197452024-02-03T17:31:21.909Z 24 480-499 183 183 183 334 334 334 F F F
197462024-02-03T17:31:21.909Z 25 500-519 183 183 183 334 334 334 F F F
197472024-02-03T17:31:21.909Z 26 520-539 183 183 183 334 334 334 F F F
197482024-02-03T17:31:21.909Z 27 540-559 183 183 183 334 334 334 F F F
197492024-02-03T17:31:21.909Z 28 560-579 183 183 183 334 334 334 F F F
197502024-02-03T17:31:21.909Z 29 580-599 183 183 183 334 334 334 F F F
197512024-02-03T17:31:21.909ZMax gen: 183, Max flush: 334
197522024-02-03T17:31:21.909Z
197532024-02-03T17:31:21.909ZBegin loop 92
197542024-02-03T17:31:21.909ZDownstairs to restart: 2
197552024-02-03T17:31:21.936Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.110567186Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871}
197562024-02-03T17:31:21.936Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111110938Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871}
197572024-02-03T17:31:21.949Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111184626Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871}
197582024-02-03T17:31:21.950Z{"msg":"Crucible 1614ceb6-fd0f-4cf9-99fd-c625f48a50dd has session id: 2428f6ca-8265-4186-9674-64d0ca847f32","v":0,"name":"crucible","level":30Crucible runtime is spawned
197592024-02-03T17:31:21.950Z,"time":"2024-02-03T17:31:21.111261598Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197602024-02-03T17:31:21.950Z{"msg":"Upstairs opts: Upstairs UUID: 1614ceb6-fd0f-4cf9-99fd-c625f48a50dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111456999Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197612024-02-03T17:31:21.950Z{"msg":"Crucible stats registered with UUID: 1614ceb6-fd0f-4cf9-99fd-c625f48a50dd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111531281Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197622024-02-03T17:31:21.950Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111599146Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871}
197632024-02-03T17:31:21.950Z{"msg":"1614ceb6-fd0f-4cf9-99fd-c625f48a50dd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111761049Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197642024-02-03T17:31:21.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111834097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197652024-02-03T17:31:21.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111922539Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197662024-02-03T17:31:21.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.111987448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197672024-02-03T17:31:21.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.112016725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197682024-02-03T17:31:21.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.112078194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197692024-02-03T17:31:21.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.112237973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197702024-02-03T17:31:21.951Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.112284973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197712024-02-03T17:31:21.951Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.112333056Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197722024-02-03T17:31:21.951Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.11240738Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197732024-02-03T17:31:21.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.112455147Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197742024-02-03T17:31:21.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.112551312Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197752024-02-03T17:31:21.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.11259547Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197762024-02-03T17:31:21.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.113795837Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197772024-02-03T17:31:21.951Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.113887855Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197782024-02-03T17:31:21.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.113961749Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197792024-02-03T17:31:21.951Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.114070619Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197802024-02-03T17:31:21.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.114645853Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197812024-02-03T17:31:21.951Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.114730567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197822024-02-03T17:31:21.952Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.114818442Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197832024-02-03T17:31:21.952Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.114860878Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197842024-02-03T17:31:21.952Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.115578425Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197852024-02-03T17:31:21.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.115765229Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197862024-02-03T17:31:21.952Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.115836267Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197872024-02-03T17:31:21.952Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.115882083Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197882024-02-03T17:31:21.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.115945343Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197892024-02-03T17:31:21.952Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.115989318Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197902024-02-03T17:31:21.952Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116046205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197912024-02-03T17:31:21.952Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116483227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197922024-02-03T17:31:21.952Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116546008Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197932024-02-03T17:31:21.952Z{"msg":"[0]R flush_numbers[0..12]: [334, 333, 329, 334, 334, 332, 328, 334, 334, 334, 332, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116575412Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197942024-02-03T17:31:21.952Z{"msg":"[0]R generation[0..12]: [183, 182, 180, 183, 183, 181, 179, 183, 183, 183, 181, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116626221Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197952024-02-03T17:31:21.953Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116658388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197962024-02-03T17:31:21.953Z{"msg":"[1]R flush_numbers[0..12]: [334, 333, 329, 334, 334, 332, 328, 334, 334, 334, 332, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116705332Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197972024-02-03T17:31:21.953Z{"msg":"[1]R generation[0..12]: [183, 182, 180, 183, 183, 181, 179, 183, 183, 183, 181, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116736981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197982024-02-03T17:31:21.953Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116787103Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
197992024-02-03T17:31:21.953Z{"Wait for a query_work_queue command to finish before sending IO
198002024-02-03T17:31:21.953Zmsg":"[2]R flush_numbers[0..12]: [334, 333, 329, 334, 334, 332, 328, 334, 334, 334, 332, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116820936Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198012024-02-03T17:31:21.953Z{"msg":"[2]R generation[0..12]: [183, 182, 180, 183, 183, 181, 179, 183, 183, 183, 181, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.11688023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198022024-02-03T17:31:21.953Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116917614Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198032024-02-03T17:31:21.953Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.116982875Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198042024-02-03T17:31:21.953Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117013105Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198052024-02-03T17:31:21.953Z{"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117073353Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198062024-02-03T17:31:21.953Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117104226Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198072024-02-03T17:31:21.954Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117161567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198082024-02-03T17:31:21.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117185912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198092024-02-03T17:31:21.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117243332Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198102024-02-03T17:31:21.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
198112024-02-03T17:31:21.954Z,"time":"2024-02-03T17:31:21.11727384Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198122024-02-03T17:31:21.954Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117406112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198132024-02-03T17:31:21.954Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117465974Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198142024-02-03T17:31:21.954Z{"msg":"1614ceb6-fd0f-4cf9-99fd-c625f48a50dd is now active with session: 2428f6ca-8265-4186-9674-64d0ca847f32","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117499157Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198152024-02-03T17:31:21.954Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117551504Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871,"session_id":"2428f6ca-8265-4186-9674-64d0ca847f32"}
198162024-02-03T17:31:21.954Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:21.117580592Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871}
198172024-02-03T17:31:21.954ZLoading write count information from file "/tmp/test_repair_verify.data"
198182024-02-03T17:31:21.954ZRun Repair workload
198192024-02-03T17:31:21.955Z01/30 Write block 35 len 20480 data: 9 9 10 14 15
198202024-02-03T17:31:21.965Z02/30 Read block 245 len 12288
198212024-02-03T17:31:21.965Z03/30 Flush
198222024-02-03T17:31:21.965Z04/30 Write block 593 len 12288 data: 13 11 11
198232024-02-03T17:31:21.965Z05/30 Read block 371 len 12288
198242024-02-03T17:31:21.965Z06/30 Flush
198252024-02-03T17:31:21.965Z07/30 Write block 384 len 32768 data: 12 11 9 10 11 12 15 16
198262024-02-03T17:31:21.965Z08/30 Read block 120 len 40960
198272024-02-03T17:31:21.965Z09/30 Read block 60 len 16384
198282024-02-03T17:31:21.966Z10/30 Write block 551 len 32768 data: 14 14 15 14 13 15 14 16
198292024-02-03T17:31:21.966Z11/30 Write block 158 len 16384 data: 16 15 15 16
198302024-02-03T17:31:21.966Z12/30 Write block 401 len 20480 data: 11 11 14 12 17
198312024-02-03T17:31:21.973Z13/30 Write block 246 len 12288 data: 10 13 15
198322024-02-03T17:31:21.973Z14/30 Write block 539 len 32768 data: 11 8 8 10 11 14 15 13
198332024-02-03T17:31:21.974Z15/30 Read block 183 len 24576
198342024-02-03T17:31:21.974Z16/30 Write block 265 len 4096 data: 14
198352024-02-03T17:31:21.974Z17/30 Write block 217 len 20480 data: 12 12 14 13 18
198362024-02-03T17:31:21.974Z18/30 Write block 551 len 24576 data: 15 15 16 15 14 16
198372024-02-03T17:31:21.974Z19/30 Read block 196 len 28672
198382024-02-03T17:31:21.974Z20/30 Read block 237 len 16384
198392024-02-03T17:31:21.982Z21/30 Flush
198402024-02-03T17:31:21.982Z22/30 Write block 313 len 16384 data: 10 9 9 9
198412024-02-03T17:31:21.982Z23/30 Read block 472 len 12288
198422024-02-03T17:31:21.982Z24/30 Write block 449 len 12288 data: 10 9 8
198432024-02-03T17:31:21.988Z25/30 Read block 399 len 40960
198442024-02-03T17:31:21.988Z26/30 Read block 136 len 40960
198452024-02-03T17:31:21.988Z27/30 Write block 272 len 12288 data: 17 17 20
198462024-02-03T17:31:21.988Z28/30 Read block 535 len 8192
198472024-02-03T17:31:21.988Z29/30 Read block 59 len 4096
198482024-02-03T17:31:21.988Z30/30 Read block 327 len 36864
198492024-02-03T17:31:21.998Z----------------------------------------------------------------
198502024-02-03T17:31:21.998Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
198512024-02-03T17:31:21.998ZStates: Active Active Active
198522024-02-03T17:31:21.998ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198532024-02-03T17:31:21.998Z 1 Acked 1000 Write 5 Done Done Sent false
198542024-02-03T17:31:21.998Z 2 Acked 1001 Read 3 Done Done Sent false
198552024-02-03T17:31:21.998Z 3 Acked 1002 Flush 0 Done Done Sent false
198562024-02-03T17:31:21.998Z 4 Acked 1003 Write 3 Done Done Sent false
198572024-02-03T17:31:21.998Z 5 Acked 1004 Read 3 Done Done Sent false
198582024-02-03T17:31:21.998Z 6 Acked 1005 Flush 0 Done Done Sent false
198592024-02-03T17:31:21.998Z 7 Acked 1006 Write 8 Done Done Sent false
198602024-02-03T17:31:21.998Z 8 Acked 1007 Read 10 Done Done Sent false
198612024-02-03T17:31:21.999Z 9 Acked 1008 Read 4 Done Done Sent false
198622024-02-03T17:31:21.999Z 10 Acked 1009 Write 8 Done Done Sent false
198632024-02-03T17:31:21.999Z 11 Acked 1010 Write 4 Done Done Sent false
198642024-02-03T17:31:21.999Z 12 Acked 1011 Write 5 Done Done Sent false
198652024-02-03T17:31:21.999Z 13 Acked 1012 Write 3 Done Done Sent false
198662024-02-03T17:31:21.999Z 14 Acked 1013 Write 8 Done Done Sent false
198672024-02-03T17:31:21.999Z 15 Acked 1014 Read 6 Done Done Sent false
198682024-02-03T17:31:21.999Z 16 Acked 1015 Write 1 Done Done Sent false
198692024-02-03T17:31:21.999Z 17 Acked 1016 Write 5 Done Done Sent false
198702024-02-03T17:31:21.999Z 18 Acked 1017 Write 6 Done Done Sent false
198712024-02-03T17:31:21.999Z 19 Acked 1018 Read 7 Done Done Sent false
198722024-02-03T17:31:21.999Z 20 Acked 1019 Read 4 Done Done Sent false
198732024-02-03T17:31:22.000Z 21 Acked 1020 Flush 0 Done Done Sent false
198742024-02-03T17:31:22.000Z 22 Acked 1021 Write 4 Done Done Sent false
198752024-02-03T17:31:22.000Z 23 Acked 1022 Read 3 Done Done Sent false
198762024-02-03T17:31:22.000Z 24 Acked 1023 Write 3 Done Done Sent false
198772024-02-03T17:31:22.000Z 25 Acked 1024 Read 10 Done Done Sent false
198782024-02-03T17:31:22.000Z 26 Acked 1025 Read 10 Done Done Sent false
198792024-02-03T17:31:22.000Z 27 Acked 1026 Write 3 Done Done Sent false
198802024-02-03T17:31:22.000Z 28 Acked 1027 Read 2 Done Done Sent false
198812024-02-03T17:31:22.000Z 29 Acked 1028 Read 1 Done Done Sent false
198822024-02-03T17:31:22.000Z 30 Acked 1029 Read 9 Done Sent Sent false
198832024-02-03T17:31:22.000Z STATES DS:0 DS:1 DS:2 TOTAL
198842024-02-03T17:31:22.000Z New 0 0 0 0
198852024-02-03T17:31:22.000Z Sent 0 1 30 31
198862024-02-03T17:31:22.000Z Done 30 29 0 59
198872024-02-03T17:31:22.001Z Skipped 0 0 0 0
198882024-02-03T17:31:22.001Z Error 0 0 0 0
198892024-02-03T17:31:22.001ZLast Flush: 1020 1020 0
198902024-02-03T17:31:22.001ZDownstairs last five completed:
198912024-02-03T17:31:22.001ZUpstairs last five completed: 30 29 28 27 26
198922024-02-03T17:31:22.001Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:21.166693063Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1871}
198932024-02-03T17:31:22.001ZWrote out file "/tmp/test_repair_verify.data"
198942024-02-03T17:31:22.009Z
198952024-02-03T17:31:22.009ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198962024-02-03T17:31:22.036ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198972024-02-03T17:31:22.036Z 0 000-019 183 183 183 334 334 334 F F F
198982024-02-03T17:31:22.044Z 1 020-039 184 184 182 335 335 333 F F F <---
198992024-02-03T17:31:22.044Z 2 040-059 180 180 180 329 329 329 F F F
199002024-02-03T17:31:22.044Z 3 060-079 183 183 183 334 334 334 F F F
199012024-02-03T17:31:22.044Z 4 080-099 183 183 183 334 334 334 F F F
199022024-02-03T17:31:22.044Z 5 100-119 181 181 181 332 332 332 F F F
199032024-02-03T17:31:22.044Z 6 120-139 179 179 179 328 328 328 F F F
199042024-02-03T17:31:22.044Z 7 140-159 184 184 183 337 337 334 F F F <---
199052024-02-03T17:31:22.044Z 8 160-179 184 184 183 337 337 334 F F F <---
199062024-02-03T17:31:22.045Z 9 180-199 183 183 183 334 334 334 F F F
199072024-02-03T17:31:22.045Z 10 200-219 184 184 181 337 337 332 F F F <---
199082024-02-03T17:31:22.045Z 11 220-239 184 184 167 337 337 307 F F F <---
199092024-02-03T17:31:22.045Z 12 240-259 184 184 178 337 337 327 F F F <---
199102024-02-03T17:31:22.045Z 13 260-279 184 184 176 337 337 321 T T F <---
199112024-02-03T17:31:22.045Z 14 280-299 183 183 183 334 334 334 F F F
199122024-02-03T17:31:22.045Z 15 300-319 182 182 182 333 333 333 T T F <---
199132024-02-03T17:31:22.045Z 16 320-339 177 177 177 323 323 323 F F F
199142024-02-03T17:31:22.045Z 17 340-359 180 180 180 329 329 329 F F F
199152024-02-03T17:31:22.045Z 18 360-379 180 180 180 330 330 330 F F F
199162024-02-03T17:31:22.045Z 19 380-399 184 184 183 337 337 334 F F F <---
199172024-02-03T17:31:22.045Z 20 400-419 184 184 183 337 337 334 F F F <---
199182024-02-03T17:31:22.045Z 21 420-439 176 176 176 322 322 322 F F F
199192024-02-03T17:31:22.046Z 22 440-459 177 177 177 323 323 323 T T F <---
199202024-02-03T17:31:22.046Z 23 460-479 182 182 182 333 333 333 F F F
199212024-02-03T17:31:22.046Z 24 480-499 183 183 183 334 334 334 F F F
199222024-02-03T17:31:22.046Z 25 500-519 183 183 183 334 334 334 F F F
199232024-02-03T17:31:22.046Z 26 520-539 184 184 183 337 337 334 F F F <---
199242024-02-03T17:31:22.046Z 27 540-559 184 184 183 337 337 334 F F F <---
199252024-02-03T17:31:22.046Z 28 560-579 183 183 183 334 334 334 F F F
199262024-02-03T17:31:22.046Z 29 580-599 184 184 183 336 336 334 F F F <---
199272024-02-03T17:31:22.046ZMax gen: 184, Max flush: 337
199282024-02-03T17:31:22.046ZError: Difference in extent metadata found!
199292024-02-03T17:31:22.046ZA Difference in extent metadata is expected here
199302024-02-03T17:31:22.046ZOn loop 92
199312024-02-03T17:31:22.047Z
199322024-02-03T17:31:22.047ZVerifying data now
199332024-02-03T17:31:27.265ZNo change after verify
199342024-02-03T17:31:27.265ZLoop: 92 Downstairs dump after verify (and repair):
199352024-02-03T17:31:27.284ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199362024-02-03T17:31:27.284Z 0 000-019 183 183 183 334 334 334 F F F
199372024-02-03T17:31:27.287Z 1 020-039 184 184 184 335 335 335 F F F
199382024-02-03T17:31:27.287Z 2 040-059 180 180 180 329 329 329 F F F
199392024-02-03T17:31:27.288Z 3 060-079 183 183 183 334 334 334 F F F
199402024-02-03T17:31:27.288Z 4 080-099 183 183 183 334 334 334 F F F
199412024-02-03T17:31:27.288Z 5 100-119 181 181 181 332 332 332 F F F
199422024-02-03T17:31:27.288Z 6 120-139 179 179 179 328 328 328 F F F
199432024-02-03T17:31:27.288Z 7 140-159 184 184 184 337 337 337 F F F
199442024-02-03T17:31:27.288Z 8 160-179 184 184 184 337 337 337 F F F
199452024-02-03T17:31:27.288Z 9 180-199 183 183 183 334 334 334 F F F
199462024-02-03T17:31:27.288Z 10 200-219 184 184 184 337 337 337 F F F
199472024-02-03T17:31:27.288Z 11 220-239 184 184 184 337 337 337 F F F
199482024-02-03T17:31:27.288Z 12 240-259 184 184 184 337 337 337 F F F
199492024-02-03T17:31:27.288Z 13 260-279 185 185 185 338 338 338 F F F
199502024-02-03T17:31:27.288Z 14 280-299 183 183 183 334 334 334 F F F
199512024-02-03T17:31:27.288Z 15 300-319 185 185 185 338 338 338 F F F
199522024-02-03T17:31:27.288Z 16 320-339 177 177 177 323 323 323 F F F
199532024-02-03T17:31:27.288Z 17 340-359 180 180 180 329 329 329 F F F
199542024-02-03T17:31:27.288Z 18 360-379 180 180 180 330 330 330 F F F
199552024-02-03T17:31:27.288Z 19 380-399 184 184 184 337 337 337 F F F
199562024-02-03T17:31:27.288Z 20 400-419 184 184 184 337 337 337 F F F
199572024-02-03T17:31:27.288Z 21 420-439 176 176 176 322 322 322 F F F
199582024-02-03T17:31:27.288Z 22 440-459 185 185 185 338 338 338 F F F
199592024-02-03T17:31:27.288Z 23 460-479 182 182 182 333 333 333 F F F
199602024-02-03T17:31:27.288Z 24 480-499 183 183 183 334 334 334 F F F
199612024-02-03T17:31:27.288Z 25 500-519 183 183 183 334 334 334 F F F
199622024-02-03T17:31:27.288Z 26 520-539 184 184 184 337 337 337 F F F
199632024-02-03T17:31:27.289Z 27 540-559 184 184 184 337 337 337 F F F
199642024-02-03T17:31:27.289Z 28 560-579 183 183 183 334 334 334 F F F
199652024-02-03T17:31:27.289Z 29 580-599 184 184 184 336 336 336 F F F
199662024-02-03T17:31:27.289ZMax gen: 185, Max flush: 338
199672024-02-03T17:31:27.292Z
199682024-02-03T17:31:27.292ZBegin loop 93
199692024-02-03T17:31:27.292ZDownstairs to restart: 2
199702024-02-03T17:31:27.322Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.499583855Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879}
199712024-02-03T17:31:27.322Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500165574Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879}
199722024-02-03T17:31:27.322Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500201187Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879}
199732024-02-03T17:31:27.326Z{"msg":"Crucible 4a156cab-5c8c-4aff-a666-a0b2f4af7200 has session id: a1e079d7-f633-4dc6-afff-f31589c6bc41","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500271935Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199742024-02-03T17:31:27.326Z{"msg":"Upstairs opts: Upstairs UUID: 4a156cab-5c8c-4aff-a666-a0b2f4af7200, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500348085Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199752024-02-03T17:31:27.326Z{"msg":"Crucible stats registered with UUID: 4a156cab-5c8c-4aff-a666-a0b2f4af7200","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500376747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199762024-02-03T17:31:27.326ZCrucible runtime is spawned
199772024-02-03T17:31:27.326Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500610306Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879}
199782024-02-03T17:31:27.326Z{"msg":"4a156cab-5c8c-4aff-a666-a0b2f4af7200 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500663104Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199792024-02-03T17:31:27.326Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500687065Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199802024-02-03T17:31:27.327Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500771796Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199812024-02-03T17:31:27.327Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500804818Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199822024-02-03T17:31:27.327Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500869392Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199832024-02-03T17:31:27.327Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.500947035Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199842024-02-03T17:31:27.327Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.501007943Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199852024-02-03T17:31:27.327Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.501152329Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199862024-02-03T17:31:27.327Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.501298249Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199872024-02-03T17:31:27.327Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.501408308Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199882024-02-03T17:31:27.327Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.50145475Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199892024-02-03T17:31:27.327Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.501607428Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199902024-02-03T17:31:27.327Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.50169854Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199912024-02-03T17:31:27.327Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.502656607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199922024-02-03T17:31:27.327Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.502791087Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199932024-02-03T17:31:27.327Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.50289961Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199942024-02-03T17:31:27.327Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.502944022Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199952024-02-03T17:31:27.327Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.50355643Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199962024-02-03T17:31:27.327Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.503631391Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199972024-02-03T17:31:27.327Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.503745623Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199982024-02-03T17:31:27.327Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.503800535Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
199992024-02-03T17:31:27.332Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.504457913Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200002024-02-03T17:31:27.332Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.504734271Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200012024-02-03T17:31:27.332Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.504831327Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200022024-02-03T17:31:27.332Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.504904646Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200032024-02-03T17:31:27.332Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.504935699Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200042024-02-03T17:31:27.332Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.504993191Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200052024-02-03T17:31:27.332Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505024184Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200062024-02-03T17:31:27.332Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505266962Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200072024-02-03T17:31:27.332Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505328138Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200082024-02-03T17:31:27.332Z{"msg":"[0]R flush_numbers[0..12]: [334, 335, 329, 334, 334, 332, 328, 337, 337, 334, 337, 337]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505352261Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200092024-02-03T17:31:27.332Z{"msg":"[0]R generation[0..12]: [183, 184, 180, 183, 183, 181, 179, 184, 184, 183, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505388798Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200102024-02-03T17:31:27.332Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.50545568Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200112024-02-03T17:31:27.332Z{"msg":"[1]R flush_numbers[0..12]: [334, 335, 329, 334, 334, 332, 328, 337, 337, 334, 337, 337]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505490922Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200122024-02-03T17:31:27.332Z{"msg":"[1]R generation[0..12]: [183, 184, 180, 183, 183, 181, 179, 184, 184, 183, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505546227Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200132024-02-03T17:31:27.332Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505575459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200142024-02-03T17:31:27.332Z{"msg":"[2]R flush_numbers[0..12]: [334, 335, 329, 334, 334, 332, 328, 337, 337, 334, 337, 337]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
200152024-02-03T17:31:27.332Z,"time":"2024-02-03T17:31:26.505653416Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200162024-02-03T17:31:27.332Z{"msg":"[2]R generation[0..12]: [183, 184, 180, 183, 183, 181, 179, 184, 184, 183, 184, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505720742Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200172024-02-03T17:31:27.332Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505749182Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200182024-02-03T17:31:27.332Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505806104Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200192024-02-03T17:31:27.333Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505836641Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200202024-02-03T17:31:27.333Z{"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505896619Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200212024-02-03T17:31:27.333Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505922343Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200222024-02-03T17:31:27.333Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.505986323Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200232024-02-03T17:31:27.333Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.506011167Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200242024-02-03T17:31:27.333Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.506040325Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200252024-02-03T17:31:27.333Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.506099745Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200262024-02-03T17:31:27.333Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.506125508Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200272024-02-03T17:31:27.333Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.506146435Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200282024-02-03T17:31:27.333Z{"msg":"4a156cab-5c8c-4aff-a666-a0b2f4af7200 is now active with session: a1e079d7-f633-4dc6-afff-f31589c6bc41","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.506188567Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200292024-02-03T17:31:27.333Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.506210448Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879,"session_id":"a1e079d7-f633-4dc6-afff-f31589c6bc41"}
200302024-02-03T17:31:27.333Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:26.506231896Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879}
200312024-02-03T17:31:27.333ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200322024-02-03T17:31:27.333ZLoading write count information from file "/tmp/test_repair_verify.data"
200332024-02-03T17:31:27.333ZRun Repair workload
200342024-02-03T17:31:27.333Z01/30 Write block 542 len 36864 data: 11 12 15 16 14 14 20 20 17
200352024-02-03T17:31:27.337Z02/30 Read block 478 len 12288
200362024-02-03T17:31:27.337Z03/30 Flush
200372024-02-03T17:31:27.337Z04/30 Write block 363 len 16384 data: 19 16 14 11
200382024-02-03T17:31:27.343Z05/30 Read block 31 len 40960
200392024-02-03T17:31:27.343Z06/30 Write block 564 len 32768 data: 11 14 16 16 13 12 13 12
200402024-02-03T17:31:27.348Z07/30 Flush
200412024-02-03T17:31:27.348Z08/30 Flush
200422024-02-03T17:31:27.348Z09/30 Write block 506 len 32768 data: 26 25 22 22 17 15 15 14
200432024-02-03T17:31:27.348Z10/30 Write block 488 len 16384 data: 14 16 17 16
200442024-02-03T17:31:27.352Z11/30 Read block 37 len 12288
200452024-02-03T17:31:27.352Z12/30 Read block 69 len 28672
200462024-02-03T17:31:27.352Z13/30 Flush
200472024-02-03T17:31:27.355Z14/30 Write block 127 len 36864 data: 14 17 19 20 19 19 14 17 18
200482024-02-03T17:31:27.355Z15/30 Read block 166 len 12288
200492024-02-03T17:31:27.358Z16/30 Read block 389 len 28672
200502024-02-03T17:31:27.358Z17/30 Read block 326 len 28672
200512024-02-03T17:31:27.361Z18/30 Write block 500 len 36864 data: 21 23 20 20 20 23 27 26 23
200522024-02-03T17:31:27.361Z19/30 Read block 436 len 32768
200532024-02-03T17:31:27.369Z20/30 Write block 134 len 20480 data: 18 19 19 20 20
200542024-02-03T17:31:27.372Z21/30 Read block 348 len 24576
200552024-02-03T17:31:27.372Z22/30 Write block 171 len 12288 data: 15 11 11
200562024-02-03T17:31:27.372Z23/30 Read block 274 len 16384
200572024-02-03T17:31:27.372Z24/30 Write block 29 len 40960 data: 11 9 11 10 8 10 10 10 11 15
200582024-02-03T17:31:27.372Z25/30 Read block 472 len 32768
200592024-02-03T17:31:27.375Z26/30 Read block 322 len 8192
200602024-02-03T17:31:27.375Z27/30 Write block 79 len 40960 data: 11 12 11 14 15 16 11 11 10 8
200612024-02-03T17:31:27.376Z28/30 Read block 327 len 12288
200622024-02-03T17:31:27.379Z29/30 Write block 233 len 32768 data: 14 14 15 14 16 12 14 11
200632024-02-03T17:31:27.379Z30/30 Read block 232 len 24576
200642024-02-03T17:31:27.383Z----------------------------------------------------------------
200652024-02-03T17:31:27.383Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
200662024-02-03T17:31:27.383ZStates: Active Active Active
200672024-02-03T17:31:27.383ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200682024-02-03T17:31:27.383Z 1 Acked 1000 Write 9 Done Done Done false
200692024-02-03T17:31:27.383Z 2 Acked 1001 Read 3 Done Done Done false
200702024-02-03T17:31:27.383Z 3 Acked 1002 Flush 0 Done Done Sent false
200712024-02-03T17:31:27.383Z 4 Acked 1003 Write 4 Done Done Sent false
200722024-02-03T17:31:27.383Z 5 Acked 1004 Read 10 Done Done Sent false
200732024-02-03T17:31:27.383Z 6 Acked 1005 Write 8 Done Done Sent false
200742024-02-03T17:31:27.383Z 7 Acked 1006 Flush 0 Done Done Sent false
200752024-02-03T17:31:27.383Z 8 Acked 1007 Flush 0 Done Done Sent false
200762024-02-03T17:31:27.383Z 9 Acked 1008 Write 8 Done Done Sent false
200772024-02-03T17:31:27.384Z 10 Acked 1009 Write 4 Done Done Sent false
200782024-02-03T17:31:27.384Z 11 Acked 1010 Read 3 Done Done Sent false
200792024-02-03T17:31:27.384Z 12 Acked 1011 Read 7 Done Done Sent false
200802024-02-03T17:31:27.384Z 13 Acked 1012 Flush 0 Done Done Sent false
200812024-02-03T17:31:27.384Z 14 Acked 1013 Write 9 Done Done Sent false
200822024-02-03T17:31:27.384Z 15 Acked 1014 Read 3 Done Done Sent false
200832024-02-03T17:31:27.384Z 16 Acked 1015 Read 7 Done Done Sent false
200842024-02-03T17:31:27.384Z 17 Acked 1016 Read 7 Done Done Sent false
200852024-02-03T17:31:27.384Z 18 Acked 1017 Write 9 Done Done Sent false
200862024-02-03T17:31:27.384Z 19 Acked 1018 Read 8 Done Done Sent false
200872024-02-03T17:31:27.384Z 20 Acked 1019 Write 5 Done Done Sent false
200882024-02-03T17:31:27.384Z 21 Acked 1020 Read 6 Done Done Sent false
200892024-02-03T17:31:27.384Z 22 Acked 1021 Write 3 Done Done Sent false
200902024-02-03T17:31:27.384Z 23 Acked 1022 Read 4 Done Done Sent false
200912024-02-03T17:31:27.384Z 24 Acked 1023 Write 10 Done Done Sent false
200922024-02-03T17:31:27.384Z 25 Acked 1024 Read 8 Done Done Sent false
200932024-02-03T17:31:27.384Z 26 Acked 1025 Read 2 Done Done Sent false
200942024-02-03T17:31:27.384Z 27 Acked 1026 Write 10 Done Done Sent false
200952024-02-03T17:31:27.384Z 28 Acked 1027 Read 3 Done Done Sent false
200962024-02-03T17:31:27.384Z 29 Acked 1028 Write 8 Done Done Sent false
200972024-02-03T17:31:27.384Z 30 Acked 1029 Read 6 Done Sent Sent false
200982024-02-03T17:31:27.384Z STATES DS:0 DS:1 DS:2 TOTAL
200992024-02-03T17:31:27.384Z New 0 0 0 0
201002024-02-03T17:31:27.384Z Sent 0 1 28 29
201012024-02-03T17:31:27.385Z Done 30 29 2 61
201022024-02-03T17:31:27.385Z Skipped 0 0 0 0
201032024-02-03T17:31:27.385Z Error 0 0 0 0
201042024-02-03T17:31:27.385ZLast Flush: 1012 1012 0
201052024-02-03T17:31:27.385ZDownstairs last five completed:
201062024-02-03T17:31:27.385ZUpstairs last five completed: 30 29 28 27 26
201072024-02-03T17:31:27.385Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:26.559342661Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1879}
201082024-02-03T17:31:27.385ZWrote out file "/tmp/test_repair_verify.data"
201092024-02-03T17:31:27.389Z
201102024-02-03T17:31:27.397ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
201112024-02-03T17:31:27.426ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201122024-02-03T17:31:27.426Z 0 000-019 183 183 183 334 334 334 F F F
201132024-02-03T17:31:27.426Z 1 020-039 184 184 184 335 335 335 T T F <---
201142024-02-03T17:31:27.426Z 2 040-059 180 180 180 329 329 329 F F F
201152024-02-03T17:31:27.426Z 3 060-079 183 183 183 334 334 334 T T F <---
201162024-02-03T17:31:27.430Z 4 080-099 183 183 183 334 334 334 T T F <---
201172024-02-03T17:31:27.430Z 5 100-119 181 181 181 332 332 332 F F F
201182024-02-03T17:31:27.430Z 6 120-139 179 179 179 328 328 328 T T F <---
201192024-02-03T17:31:27.430Z 7 140-159 184 184 184 337 337 337 F F F
201202024-02-03T17:31:27.430Z 8 160-179 184 184 184 337 337 337 T T F <---
201212024-02-03T17:31:27.430Z 9 180-199 183 183 183 334 334 334 F F F
201222024-02-03T17:31:27.430Z 10 200-219 184 184 184 337 337 337 F F F
201232024-02-03T17:31:27.430Z 11 220-239 184 184 184 337 337 337 T T F <---
201242024-02-03T17:31:27.430Z 12 240-259 184 184 184 337 337 337 T T F <---
201252024-02-03T17:31:27.430Z 13 260-279 185 185 185 338 338 338 F F F
201262024-02-03T17:31:27.431Z 14 280-299 183 183 183 334 334 334 F F F
201272024-02-03T17:31:27.431Z 15 300-319 185 185 185 338 338 338 F F F
201282024-02-03T17:31:27.431Z 16 320-339 177 177 177 323 323 323 F F F
201292024-02-03T17:31:27.431Z 17 340-359 180 180 180 329 329 329 F F F
201302024-02-03T17:31:27.431Z 18 360-379 186 186 180 340 340 330 F F F <---
201312024-02-03T17:31:27.431Z 19 380-399 184 184 184 337 337 337 F F F
201322024-02-03T17:31:27.431Z 20 400-419 184 184 184 337 337 337 F F F
201332024-02-03T17:31:27.431Z 21 420-439 176 176 176 322 322 322 F F F
201342024-02-03T17:31:27.431Z 22 440-459 185 185 185 338 338 338 F F F
201352024-02-03T17:31:27.431Z 23 460-479 182 182 182 333 333 333 F F F
201362024-02-03T17:31:27.431Z 24 480-499 186 186 183 342 342 334 F F F <---
201372024-02-03T17:31:27.431Z 25 500-519 186 186 183 342 342 334 T T F <---
201382024-02-03T17:31:27.431Z 26 520-539 184 184 184 337 337 337 F F F
201392024-02-03T17:31:27.431Z 27 540-559 186 186 184 339 339 337 F F T <---
201402024-02-03T17:31:27.431Z 28 560-579 186 186 183 340 340 334 F F F <---
201412024-02-03T17:31:27.431Z 29 580-599 184 184 184 336 336 336 F F F
201422024-02-03T17:31:27.431ZMax gen: 186, Max flush: 342
201432024-02-03T17:31:27.431ZError: Difference in extent metadata found!
201442024-02-03T17:31:27.441ZA Difference in extent metadata is expected here
201452024-02-03T17:31:27.441ZOn loop 93
201462024-02-03T17:31:27.441Z
201472024-02-03T17:31:27.444ZVerifying data now
201482024-02-03T17:31:32.535ZNo change after verify
201492024-02-03T17:31:32.535ZLoop: 93 Downstairs dump after verify (and repair):
201502024-02-03T17:31:32.560ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201512024-02-03T17:31:32.560Z 0 000-019 183 183 183 334 334 334 F F F
201522024-02-03T17:31:32.564Z 1 020-039 187 187 187 343 343 343 F F F
201532024-02-03T17:31:32.564Z 2 040-059 180 180 180 329 329 329 F F F
201542024-02-03T17:31:32.564Z 3 060-079 187 187 187 343 343 343 F F F
201552024-02-03T17:31:32.564Z 4 080-099 187 187 187 343 343 343 F F F
201562024-02-03T17:31:32.564Z 5 100-119 181 181 181 332 332 332 F F F
201572024-02-03T17:31:32.564Z 6 120-139 187 187 187 343 343 343 F F F
201582024-02-03T17:31:32.564Z 7 140-159 184 184 184 337 337 337 F F F
201592024-02-03T17:31:32.564Z 8 160-179 187 187 187 343 343 343 F F F
201602024-02-03T17:31:32.565Z 9 180-199 183 183 183 334 334 334 F F F
201612024-02-03T17:31:32.565Z 10 200-219 184 184 184 337 337 337 F F F
201622024-02-03T17:31:32.565Z 11 220-239 187 187 187 343 343 343 F F F
201632024-02-03T17:31:32.565Z 12 240-259 187 187 187 343 343 343 F F F
201642024-02-03T17:31:32.565Z 13 260-279 185 185 185 338 338 338 F F F
201652024-02-03T17:31:32.565Z 14 280-299 183 183 183 334 334 334 F F F
201662024-02-03T17:31:32.565Z 15 300-319 185 185 185 338 338 338 F F F
201672024-02-03T17:31:32.565Z 16 320-339 177 177 177 323 323 323 F F F
201682024-02-03T17:31:32.565Z 17 340-359 180 180 180 329 329 329 F F F
201692024-02-03T17:31:32.565Z 18 360-379 186 186 186 340 340 340 F F F
201702024-02-03T17:31:32.565Z 19 380-399 184 184 184 337 337 337 F F F
201712024-02-03T17:31:32.565Z 20 400-419 184 184 184 337 337 337 F F F
201722024-02-03T17:31:32.565Z 21 420-439 176 176 176 322 322 322 F F F
201732024-02-03T17:31:32.565Z 22 440-459 185 185 185 338 338 338 F F F
201742024-02-03T17:31:32.565Z 23 460-479 182 182 182 333 333 333 F F F
201752024-02-03T17:31:32.565Z 24 480-499 186 186 186 342 342 342 F F F
201762024-02-03T17:31:32.565Z 25 500-519 187 187 187 343 343 343 F F F
201772024-02-03T17:31:32.565Z 26 520-539 184 184 184 337 337 337 F F F
201782024-02-03T17:31:32.565Z 27 540-559 186 186 186 339 339 339 F F F
201792024-02-03T17:31:32.565Z 28 560-579 186 186 186 340 340 340 F F F
201802024-02-03T17:31:32.565Z 29 580-599 184 184 184 336 336 336 F F F
201812024-02-03T17:31:32.565ZMax gen: 187, Max flush: 343
201822024-02-03T17:31:32.565Z
201832024-02-03T17:31:32.565ZBegin loop 94
201842024-02-03T17:31:32.565ZDownstairs to restart: 0
201852024-02-03T17:31:32.597Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.775552343Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887}
201862024-02-03T17:31:32.605Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776067926Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887}
201872024-02-03T17:31:32.605Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776184482Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887}
201882024-02-03T17:31:32.605Z{"msg":"Crucible 53593413-1874-4252-83a6-3c2c31d4ce5b has session id: c5e2954d-638f-4743-b3c9-f999bd033244","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776225551Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201892024-02-03T17:31:32.605Z{"msg":"Upstairs opts: Upstairs UUID: 53593413-1874-4252-83a6-3c2c31d4ce5b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false",Crucible runtime is spawned
201902024-02-03T17:31:32.605Z"v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776424365Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201912024-02-03T17:31:32.605Z{"msg":"Crucible stats registered with UUID: 53593413-1874-4252-83a6-3c2c31d4ce5b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776521059Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201922024-02-03T17:31:32.605Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776569015Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887}
201932024-02-03T17:31:32.605Z{"msg":"53593413-1874-4252-83a6-3c2c31d4ce5b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776712371Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201942024-02-03T17:31:32.605Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776850375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201952024-02-03T17:31:32.605Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776931522Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201962024-02-03T17:31:32.605Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.776970543Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201972024-02-03T17:31:32.605Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.777013672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201982024-02-03T17:31:32.605Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.777096025Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
201992024-02-03T17:31:32.605Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.777141612Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202002024-02-03T17:31:32.605Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.777259862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202012024-02-03T17:31:32.605Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.77733516Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202022024-02-03T17:31:32.605Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.777380596Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202032024-02-03T17:31:32.606Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.777426081Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202042024-02-03T17:31:32.606Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.777469593Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202052024-02-03T17:31:32.606Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.777511904Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202062024-02-03T17:31:32.606Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.778638728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202072024-02-03T17:31:32.606Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.778733715Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202082024-02-03T17:31:32.606Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.778783195Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202092024-02-03T17:31:32.606Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.778924485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202102024-02-03T17:31:32.606Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.779785588Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202112024-02-03T17:31:32.606Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.779863304Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202122024-02-03T17:31:32.606Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.779891689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202132024-02-03T17:31:32.606Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.779981731Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202142024-02-03T17:31:32.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.780769331Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202152024-02-03T17:31:32.606Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.780869524Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202162024-02-03T17:31:32.606Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.780898149Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202172024-02-03T17:31:32.606Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.780921116Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202182024-02-03T17:31:32.606Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.780963392Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202192024-02-03T17:31:32.606Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.780987785Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202202024-02-03T17:31:32.606Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.781252817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202212024-02-03T17:31:32.611Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.781997117Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202222024-02-03T17:31:32.611Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782074095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202232024-02-03T17:31:32.611Z{"msg":"[0]R flush_numbers[0..12]: [334, 343, 329, 343, 343, 332, 343, 337, 343, 334, 337, 343]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782101821Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202242024-02-03T17:31:32.611Z{"msg":"[0]R generation[0..12]: [183, 187, 180, 187, 187, 181, 187, 184, 187, 183, 184, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782158507Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202252024-02-03T17:31:32.611Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782184358Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202262024-02-03T17:31:32.611Z{"msg":"[1]R flush_numbers[0..12]: [334, 343, 329, 343, 343, 332, 343, 337, 343, 334, 337, 343]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782207349Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202272024-02-03T17:31:32.611Z{"msg":"[1]R generation[0..12]: [183, 187, 180, 187, 187, 181, 187, 184, 187, 183, 184, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782265205Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202282024-02-03T17:31:32.611Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782290457Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202292024-02-03T17:31:32.611Z{"msg":"[2]R flush_numbers[0..12]: [334, 343, 329, 343, 343, 332, 343, 337, 343, 334, 337, 343]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
202302024-02-03T17:31:32.612Z,"time":"2024-02-03T17:31:31.782350972Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202312024-02-03T17:31:32.612Z{"msg":"[2]R generation[0..12]: [183, 187, 180, 187, 187, 181, 187, 184, 187, 183, 184, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.78239977Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202322024-02-03T17:31:32.612Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782425249Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202332024-02-03T17:31:32.612Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782471879Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202342024-02-03T17:31:32.612Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782495023Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202352024-02-03T17:31:32.612Z{"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782517615Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202362024-02-03T17:31:32.612Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782565647Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202372024-02-03T17:31:32.612Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782589178Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202382024-02-03T17:31:32.612Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.78263121Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202392024-02-03T17:31:32.612Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782654847Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202402024-02-03T17:31:32.612Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782677131Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202412024-02-03T17:31:32.612Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782721263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202422024-02-03T17:31:32.612Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782743665Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202432024-02-03T17:31:32.612Z{"msg":"53593413-1874-4252-83a6-3c2c31d4ce5b is now active with session: c5e2954d-638f-4743-b3c9-f999bd033244","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782764973Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202442024-02-03T17:31:32.612Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782784211Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887,"session_id":"c5e2954d-638f-4743-b3c9-f999bd033244"}
202452024-02-03T17:31:32.612Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:31.782831188Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887}
202462024-02-03T17:31:32.612ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202472024-02-03T17:31:32.612ZLoading write count information from file "/tmp/test_repair_verify.data"
202482024-02-03T17:31:32.612ZRun Repair workload
202492024-02-03T17:31:32.612Z01/30 Write block 470 len 16384 data: 13 15 13 12
202502024-02-03T17:31:32.612Z02/30 Read block 179 len 32768
202512024-02-03T17:31:32.616Z03/30 Write block 179 len 32768 data: 12 13 15 14 12 9 10 10
202522024-02-03T17:31:32.616Z04/30 Write block 452 len 16384 data: 9 10 9 12
202532024-02-03T17:31:32.616Z05/30 Flush
202542024-02-03T17:31:32.616Z06/30 Write block 329 len 32768 data: 11 13 14 13 14 14 12 11
202552024-02-03T17:31:32.622Z07/30 Read block 576 len 40960
202562024-02-03T17:31:32.623Z08/30 Read block 589 len 4096
202572024-02-03T17:31:32.623Z09/30 Write block 236 len 36864 data: 15 17 13 15 12 11 8 6 8
202582024-02-03T17:31:32.623Z10/30 Read block 451 len 28672
202592024-02-03T17:31:32.626Z11/30 Write block 290 len 20480 data: 11 12 12 14 16
202602024-02-03T17:31:32.626Z12/30 Write block 411 len 4096 data: 15
202612024-02-03T17:31:32.626Z13/30 Flush
202622024-02-03T17:31:32.630Z14/30 Write block 439 len 8192 data: 15 16
202632024-02-03T17:31:32.633Z15/30 Read block 241 len 28672
202642024-02-03T17:31:32.633Z16/30 Flush
202652024-02-03T17:31:32.642Z17/30 Write block 156 len 8192 data: 17 17
202662024-02-03T17:31:32.642Z18/30 Flush
202672024-02-03T17:31:32.645Z19/30 Write block 202 len 4096 data: 12
202682024-02-03T17:31:32.645Z20/30 Read block 582 len 36864
202692024-02-03T17:31:32.645Z21/30 Read block 451 len 28672
202702024-02-03T17:31:32.645Z22/30 Write block 469 len 12288 data: 14 14 16
202712024-02-03T17:31:32.645Z23/30 Read block 298 len 8192
202722024-02-03T17:31:32.645Z24/30 Flush
202732024-02-03T17:31:32.651Z25/30 Write block 407 len 12288 data: 16 14 16
202742024-02-03T17:31:32.651Z26/30 Read block 354 len 28672
202752024-02-03T17:31:32.651Z27/30 Read block 507 len 40960
202762024-02-03T17:31:32.655Z28/30 Read block 487 len 16384
202772024-02-03T17:31:32.655Z29/30 Write block 159 len 36864 data: 16 16 17 16 13 13 11 10 11
202782024-02-03T17:31:32.655Z30/30 Write block 460 len 16384 data: 19 20 22 20
202792024-02-03T17:31:32.655Z----------------------------------------------------------------
202802024-02-03T17:31:32.655Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
202812024-02-03T17:31:32.655ZStates: Active Active Active
202822024-02-03T17:31:32.655ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202832024-02-03T17:31:32.655Z 1 Acked 1000 Write 4 Done Done Done false
202842024-02-03T17:31:32.655Z 2 Acked 1001 Read 8 Done Done Done false
202852024-02-03T17:31:32.655Z 3 Acked 1002 Write 8 Sent Done Done false
202862024-02-03T17:31:32.655Z 4 Acked 1003 Write 4 Sent Done Done false
202872024-02-03T17:31:32.655Z 5 Acked 1004 Flush 0 Sent Done Done false
202882024-02-03T17:31:32.655Z 6 Acked 1005 Write 8 Sent Done Done false
202892024-02-03T17:31:32.655Z 7 Acked 1006 Read 10 Sent Done Done false
202902024-02-03T17:31:32.655Z 8 Acked 1007 Read 1 Sent Done Done false
202912024-02-03T17:31:32.655Z 9 Acked 1008 Write 9 Sent Done Done false
202922024-02-03T17:31:32.656Z 10 Acked 1009 Read 7 Sent Done Done false
202932024-02-03T17:31:32.656Z 11 Acked 1010 Write 5 Sent Done Done false
202942024-02-03T17:31:32.656Z 12 Acked 1011 Write 1 Sent Done Done false
202952024-02-03T17:31:32.656Z 13 Acked 1012 Flush 0 Sent Done Done false
202962024-02-03T17:31:32.656Z 14 Acked 1013 Write 2 Sent Done Done false
202972024-02-03T17:31:32.656Z 15 Acked 1014 Read 7 Sent Done Done false
202982024-02-03T17:31:32.656Z 16 Acked 1015 Flush 0 Sent Done Done false
202992024-02-03T17:31:32.656Z 17 Acked 1016 Write 2 Sent Done Done false
203002024-02-03T17:31:32.656Z 18 Acked 1017 Flush 0 Sent Done Done false
203012024-02-03T17:31:32.656Z 19 Acked 1018 Write 1 Sent Done Done false
203022024-02-03T17:31:32.656Z 20 Acked 1019 Read 9 Sent Done Done false
203032024-02-03T17:31:32.656Z 21 Acked 1020 Read 7 Sent Done Done false
203042024-02-03T17:31:32.656Z 22 Acked 1021 Write 3 Sent Done Done false
203052024-02-03T17:31:32.656Z 23 Acked 1022 Read 2 Sent Done Done false
203062024-02-03T17:31:32.656Z 24 Acked 1023 Flush 0 Sent Done Done false
203072024-02-03T17:31:32.656Z 25 Acked 1024 Write 3 Sent Done Done false
203082024-02-03T17:31:32.656Z 26 Acked 1025 Read 7 Sent Done Done false
203092024-02-03T17:31:32.656Z 27 Acked 1026 Read 10 Sent Done Done false
203102024-02-03T17:31:32.656Z 28 Acked 1027 Read 4 Sent Done Done false
203112024-02-03T17:31:32.656Z 29 Acked 1028 Write 9 Sent Sent Sent false
203122024-02-03T17:31:32.656Z 30 Acked 1029 Write 4 Sent Sent Sent false
203132024-02-03T17:31:32.656Z STATES DS:0 DS:1 DS:2 TOTAL
203142024-02-03T17:31:32.656Z New 0 0 0 0
203152024-02-03T17:31:32.656Z Sent 28 2 2 32
203162024-02-03T17:31:32.657Z Done 2 28 28 58
203172024-02-03T17:31:32.657Z Skipped 0 0 0 0
203182024-02-03T17:31:32.657Z Error 0 0 0 0
203192024-02-03T17:31:32.657ZLast Flush: 0 1023 1023
203202024-02-03T17:31:32.657ZDownstairs last five completed:
203212024-02-03T17:31:32.657ZUpstairs last five completed: 30 29 28 27 26
203222024-02-03T17:31:32.657Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:31.833046471Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1887}
203232024-02-03T17:31:32.661ZWrote out file "/tmp/test_repair_verify.data"
203242024-02-03T17:31:32.664Z
203252024-02-03T17:31:32.670ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203262024-02-03T17:31:32.697ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203272024-02-03T17:31:32.697Z 0 000-019 183 183 183 334 334 334 F F F
203282024-02-03T17:31:32.697Z 1 020-039 187 187 187 343 343 343 F F F
203292024-02-03T17:31:32.697Z 2 040-059 180 180 180 329 329 329 F F F
203302024-02-03T17:31:32.697Z 3 060-079 187 187 187 343 343 343 F F F
203312024-02-03T17:31:32.701Z 4 080-099 187 187 187 343 343 343 F F F
203322024-02-03T17:31:32.701Z 5 100-119 181 181 181 332 332 332 F F F
203332024-02-03T17:31:32.701Z 6 120-139 187 187 187 343 343 343 F F F
203342024-02-03T17:31:32.701Z 7 140-159 184 188 188 337 347 347 F T T <---
203352024-02-03T17:31:32.701Z 8 160-179 187 188 188 343 344 344 F T T <---
203362024-02-03T17:31:32.701Z 9 180-199 183 188 188 334 344 344 F F F <---
203372024-02-03T17:31:32.701Z 10 200-219 184 188 188 337 348 348 F F F <---
203382024-02-03T17:31:32.701Z 11 220-239 187 188 188 343 345 345 F F F <---
203392024-02-03T17:31:32.701Z 12 240-259 187 188 188 343 345 345 F F F <---
203402024-02-03T17:31:32.701Z 13 260-279 185 185 185 338 338 338 F F F
203412024-02-03T17:31:32.702Z 14 280-299 183 188 188 334 345 345 F F F <---
203422024-02-03T17:31:32.702Z 15 300-319 185 185 185 338 338 338 F F F
203432024-02-03T17:31:32.702Z 16 320-339 177 188 188 323 345 345 F F F <---
203442024-02-03T17:31:32.702Z 17 340-359 180 180 180 329 329 329 F F F
203452024-02-03T17:31:32.702Z 18 360-379 186 186 186 340 340 340 F F F
203462024-02-03T17:31:32.702Z 19 380-399 184 184 184 337 337 337 F F F
203472024-02-03T17:31:32.702Z 20 400-419 184 188 188 337 345 345 F T T <---
203482024-02-03T17:31:32.702Z 21 420-439 176 188 188 322 346 346 F F F <---
203492024-02-03T17:31:32.702Z 22 440-459 185 188 188 338 346 346 F F F <---
203502024-02-03T17:31:32.702Z 23 460-479 182 188 188 333 348 348 T T T <---
203512024-02-03T17:31:32.702Z 24 480-499 186 186 186 342 342 342 F F F
203522024-02-03T17:31:32.702Z 25 500-519 187 187 187 343 343 343 F F F
203532024-02-03T17:31:32.702Z 26 520-539 184 184 184 337 337 337 F F F
203542024-02-03T17:31:32.703Z 27 540-559 186 186 186 339 339 339 F F F
203552024-02-03T17:31:32.703Z 28 560-579 186 186 186 340 340 340 F F F
203562024-02-03T17:31:32.703Z 29 580-599 184 184 184 336 336 336 F F F
203572024-02-03T17:31:32.703ZMax gen: 188, Max flush: 348
203582024-02-03T17:31:32.703ZError: Difference in extent metadata found!
203592024-02-03T17:31:32.707ZA Difference in extent metadata is expected here
203602024-02-03T17:31:32.707ZOn loop 94
203612024-02-03T17:31:32.707Z
203622024-02-03T17:31:32.707ZVerifying data now
203632024-02-03T17:31:37.770ZNo change after verify
203642024-02-03T17:31:37.770ZLoop: 94 Downstairs dump after verify (and repair):
203652024-02-03T17:31:37.804ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203662024-02-03T17:31:37.804Z 0 000-019 183 183 183 334 334 334 F F F
203672024-02-03T17:31:37.814Z 1 020-039 187 187 187 343 343 343 F F F
203682024-02-03T17:31:37.815Z 2 040-059 180 180 180 329 329 329 F F F
203692024-02-03T17:31:37.815Z 3 060-079 187 187 187 343 343 343 F F F
203702024-02-03T17:31:37.815Z 4 080-099 187 187 187 343 343 343 F F F
203712024-02-03T17:31:37.815Z 5 100-119 181 181 181 332 332 332 F F F
203722024-02-03T17:31:37.815Z 6 120-139 187 187 187 343 343 343 F F F
203732024-02-03T17:31:37.815Z 7 140-159 189 189 189 349 349 349 F F F
203742024-02-03T17:31:37.815Z 8 160-179 189 189 189 349 349 349 F F F
203752024-02-03T17:31:37.815Z 9 180-199 188 188 188 344 344 344 F F F
203762024-02-03T17:31:37.816Z 10 200-219 188 188 188 348 348 348 F F F
203772024-02-03T17:31:37.816Z 11 220-239 188 188 188 345 345 345 F F F
203782024-02-03T17:31:37.816Z 12 240-259 188 188 188 345 345 345 F F F
203792024-02-03T17:31:37.816Z 13 260-279 185 185 185 338 338 338 F F F
203802024-02-03T17:31:37.816Z 14 280-299 188 188 188 345 345 345 F F F
203812024-02-03T17:31:37.816Z 15 300-319 185 185 185 338 338 338 F F F
203822024-02-03T17:31:37.816Z 16 320-339 188 188 188 345 345 345 F F F
203832024-02-03T17:31:37.816Z 17 340-359 180 180 180 329 329 329 F F F
203842024-02-03T17:31:37.817Z 18 360-379 186 186 186 340 340 340 F F F
203852024-02-03T17:31:37.817Z 19 380-399 184 184 184 337 337 337 F F F
203862024-02-03T17:31:37.817Z 20 400-419 189 189 189 349 349 349 F F F
203872024-02-03T17:31:37.817Z 21 420-439 188 188 188 346 346 346 F F F
203882024-02-03T17:31:37.817Z 22 440-459 188 188 188 346 346 346 F F F
203892024-02-03T17:31:37.817Z 23 460-479 189 189 189 349 349 349 F F F
203902024-02-03T17:31:37.818Z 24 480-499 186 186 186 342 342 342 F F F
203912024-02-03T17:31:37.818Z 25 500-519 187 187 187 343 343 343 F F F
203922024-02-03T17:31:37.818Z 26 520-539 184 184 184 337 337 337 F F F
203932024-02-03T17:31:37.818Z 27 540-559 186 186 186 339 339 339 F F F
203942024-02-03T17:31:37.818Z 28 560-579 186 186 186 340 340 340 F F F
203952024-02-03T17:31:37.818Z 29 580-599 184 184 184 336 336 336 F F F
203962024-02-03T17:31:37.826ZMax gen: 189, Max flush: 349
203972024-02-03T17:31:37.826Z
203982024-02-03T17:31:37.826ZBegin loop 95
203992024-02-03T17:31:37.826ZDownstairs to restart: 2
204002024-02-03T17:31:37.854Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.014279224Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895}
204012024-02-03T17:31:37.854Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.01480963Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895}
204022024-02-03T17:31:37.854Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.014839512Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895}
204032024-02-03T17:31:37.854Z{"msg":"Crucible f7cc1054-9d21-4306-9d0d-361cdce7087b has session id: d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.014890476Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204042024-02-03T17:31:37.854Z{"msg":"Upstairs opts: Upstairs UUID: f7cc1054-9d21-4306-9d0d-361cdce7087b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.014927468Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204052024-02-03T17:31:37.854Z{"msg":"Crucible stats registered with UUID: f7cc1054-9d21-4306-9d0d-361cdce7087b","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.014971997Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204062024-02-03T17:31:37.855ZCrucible runtime is spawned
204072024-02-03T17:31:37.855Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.015418636Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895}
204082024-02-03T17:31:37.855Z{"msg":"f7cc1054-9d21-4306-9d0d-361cdce7087b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.015602565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204092024-02-03T17:31:37.855Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.015701895Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204102024-02-03T17:31:37.855Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.015762195Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204112024-02-03T17:31:37.855Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.015851305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204122024-02-03T17:31:37.855Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.015922018Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204132024-02-03T17:31:37.855Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.01596506Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204142024-02-03T17:31:37.855Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016003737Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204152024-02-03T17:31:37.855Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016042435Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204162024-02-03T17:31:37.856Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016081998Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204172024-02-03T17:31:37.856Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016246573Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204182024-02-03T17:31:37.856Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016294961Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204192024-02-03T17:31:37.856Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:37.016373819Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204202024-02-03T17:31:37.856Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016418241Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204212024-02-03T17:31:37.856Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:37.016487016Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204222024-02-03T17:31:37.856Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:37.016564305Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204232024-02-03T17:31:37.856Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016635142Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204242024-02-03T17:31:37.856Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016666107Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204252024-02-03T17:31:37.856Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.016822376Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204262024-02-03T17:31:37.856Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.017672194Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204272024-02-03T17:31:37.857Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.017789938Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204282024-02-03T17:31:37.857Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.017833149Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204292024-02-03T17:31:37.857Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.01785735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204302024-02-03T17:31:37.857Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.018819369Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204312024-02-03T17:31:37.857Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.018935841Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204322024-02-03T17:31:37.857Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.019743457Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204332024-02-03T17:31:37.857Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.019811786Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204342024-02-03T17:31:37.858Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.01983898Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204352024-02-03T17:31:37.858Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.019899141Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204362024-02-03T17:31:37.858Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.019959337Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204372024-02-03T17:31:37.858Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:37.020018248Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204382024-02-03T17:31:38.841Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.017708324Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204392024-02-03T17:31:38.841Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.017813387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204402024-02-03T17:31:38.856Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.018938848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204412024-02-03T17:31:38.856Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.019016322Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204422024-02-03T17:31:38.856Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.019776244Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204432024-02-03T17:31:38.856Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.020652673Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204442024-02-03T17:31:38.857Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.02073272Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204452024-02-03T17:31:38.857Z{"msg":"[0]R flush_numbers[0..12]: [334, 343, 329, 343, 343, 332, 343, 349, 349, 344, 348, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.020781032Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204462024-02-03T17:31:38.857Z{"msg":"[0]R generation[0..12]: [183, 187, 180, 187, 187, 181, 187, 189, 189, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.020837662Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204472024-02-03T17:31:38.857Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.020865336Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204482024-02-03T17:31:38.857Z{"msg":"[1]R flush_numbers[0..12]: [334, 343, 329, 343, 343, 332, 343, 349, 349, 344, 348, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.020908352Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204492024-02-03T17:31:38.857Z{"msg":"[1]R generation[0..12]: [183, 187, 180, 187, 187, 181, 187, 189, 189, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.020933619Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204502024-02-03T17:31:38.857Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.020976222Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"Wait for a query_work_queue command to finish before sending IO
204512024-02-03T17:31:38.857Z}
204522024-02-03T17:31:38.857Z{"msg":"[2]R flush_numbers[0..12]: [334, 343, 329, 343, 343, 332, 343, 349, 349, 344, 348, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.02100741Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204532024-02-03T17:31:38.857Z{"msg":"[2]R generation[0..12]: [183, 187, 180, 187, 187, 181, 187, 189, 189, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021056923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204542024-02-03T17:31:38.857Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021082624Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204552024-02-03T17:31:38.857Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021137246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204562024-02-03T17:31:38.858Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021199562Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204572024-02-03T17:31:38.858Z{"msg":"Next flush: 350","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021225464Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204582024-02-03T17:31:38.858Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021280962Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204592024-02-03T17:31:38.858Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.02131335Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204602024-02-03T17:31:38.858Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021368372Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204612024-02-03T17:31:38.858Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.02143705Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204622024-02-03T17:31:38.858Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.02146497Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204632024-02-03T17:31:38.858Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021530906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204642024-02-03T17:31:38.858Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021557944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204652024-02-03T17:31:38.858Z{"msg":"f7cc1054-9d21-4306-9d0d-361cdce7087b is now active with session: d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204662024-02-03T17:31:38.858Z,"time":"2024-02-03T17:31:38.021625612Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204672024-02-03T17:31:38.858Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.02165773Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895,"session_id":"d9eba76a-bcc4-42f1-b5d9-5ca5f5db6e68"}
204682024-02-03T17:31:38.858Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:38.021715456Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895}
204692024-02-03T17:31:38.858ZLoading write count information from file "/tmp/test_repair_verify.data"
204702024-02-03T17:31:38.858ZRun Repair workload
204712024-02-03T17:31:38.858Z01/30 Write block 195 len 40960 data: 11 12 11 12 14 11 11 13 14 14
204722024-02-03T17:31:38.859Z02/30 Flush
204732024-02-03T17:31:38.878Z03/30 Write block 368 len 4096 data: 12
204742024-02-03T17:31:38.878Z04/30 Write block 314 len 28672 data: 10 10 10 9 7 14 11
204752024-02-03T17:31:38.878Z05/30 Write block 357 len 28672 data: 17 19 19 18 19 17 20
204762024-02-03T17:31:38.878Z06/30 Write block 100 len 32768 data: 16 17 17 20 18 21 19 21
204772024-02-03T17:31:38.878Z07/30 Flush
204782024-02-03T17:31:38.879Z08/30 Write block 60 len 36864 data: 16 16 15 17 17 12 13 11 9
204792024-02-03T17:31:38.879Z09/30 Write block 63 len 12288 data: 18 18 13
204802024-02-03T17:31:38.879Z10/30 Read block 529 len 8192
204812024-02-03T17:31:38.879Z11/30 Flush
204822024-02-03T17:31:38.891Z12/30 Write block 484 len 8192 data: 9 11
204832024-02-03T17:31:38.892Z13/30 Flush
204842024-02-03T17:31:38.892Z14/30 Write block 573 len 32768 data: 9 10 13 14 16 20 22 22
204852024-02-03T17:31:38.892Z15/30 Write block 346 len 24576 data: 9 11 13 12 11 11
204862024-02-03T17:31:38.892Z16/30 Read block 320 len 36864
204872024-02-03T17:31:38.893Z17/30 Read block 342 len 24576
204882024-02-03T17:31:38.893Z18/30 Write block 420 len 8192 data: 19 17
204892024-02-03T17:31:38.893Z19/30 Read block 569 len 36864
204902024-02-03T17:31:38.893Z20/30 Read block 554 len 24576
204912024-02-03T17:31:38.893Z21/30 Write block 373 len 24576 data: 11 11 11 14 13 14
204922024-02-03T17:31:38.894Z22/30 Flush
204932024-02-03T17:31:38.894Z23/30 Write block 556 len 24576 data: 17 15 17 17 13 13
204942024-02-03T17:31:38.919Z24/30 Write block 579 len 40960 data: 23 23 23 21 20 20 18 21 17 18
204952024-02-03T17:31:38.919Z25/30 Read block 589 len 40960
204962024-02-03T17:31:38.919Z26/30 Write block 43 len 40960 data: 12 11 9 8 8 10 10 13 12 12
204972024-02-03T17:31:38.920Z27/30 Read block 218 len 20480
204982024-02-03T17:31:38.920Z28/30 Write block 243 len 8192 data: 7 9
204992024-02-03T17:31:38.920Z29/30 Read block 587 len 16384
205002024-02-03T17:31:38.920Z30/30 Write block 95 len 12288 data: 13 14 14
205012024-02-03T17:31:38.920Z----------------------------------------------------------------
205022024-02-03T17:31:38.920Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:28
205032024-02-03T17:31:38.920ZStates: Active Active Active
205042024-02-03T17:31:38.920ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
205052024-02-03T17:31:38.920Z 3 Acked 1002 Write 1 Done Done Sent false
205062024-02-03T17:31:38.920Z 4 Acked 1003 Write 7 Done Done Sent false
205072024-02-03T17:31:38.920Z 5 Acked 1004 Write 7 Done Done Sent false
205082024-02-03T17:31:38.920Z 6 Acked 1005 Write 8 Done Done Sent false
205092024-02-03T17:31:38.921Z 7 Acked 1006 Flush 0 Done Done Sent false
205102024-02-03T17:31:38.921Z 8 Acked 1007 Write 9 Done Done Sent false
205112024-02-03T17:31:38.921Z 9 Acked 1008 Write 3 Done Done Sent false
205122024-02-03T17:31:38.921Z 10 Acked 1009 Read 2 Done Done Sent false
205132024-02-03T17:31:38.921Z 11 Acked 1010 Flush 0 Done Done Sent false
205142024-02-03T17:31:38.922Z 12 Acked 1011 Write 2 Done Done Sent false
205152024-02-03T17:31:38.922Z 13 Acked 1012 Flush 0 Done Done Sent false
205162024-02-03T17:31:38.922Z 14 Acked 1013 Write 8 Done Done Sent false
205172024-02-03T17:31:38.922Z 15 Acked 1014 Write 6 Done Done Sent false
205182024-02-03T17:31:38.922Z 16 Acked 1015 Read 9 Done Done Sent false
205192024-02-03T17:31:38.922Z 17 Acked 1016 Read 6 Done Done Sent false
205202024-02-03T17:31:38.923Z 18 Acked 1017 Write 2 Done Done Sent false
205212024-02-03T17:31:38.923Z 19 Acked 1018 Read 9 Done Done Sent false
205222024-02-03T17:31:38.923Z 20 Acked 1019 Read 6 Done Done Sent false
205232024-02-03T17:31:38.923Z 21 Acked 1020 Write 6 Done Done Sent false
205242024-02-03T17:31:38.923Z 22 Acked 1021 Flush 0 Done Done Sent false
205252024-02-03T17:31:38.923Z 23 Acked 1022 Write 6 Done Done Sent false
205262024-02-03T17:31:38.923Z 24 Acked 1023 Write 10 Done Done Sent false
205272024-02-03T17:31:38.924Z 25 Acked 1024 Read 10 Done Done Sent false
205282024-02-03T17:31:38.924Z 26 Acked 1025 Write 10 Done Done Sent false
205292024-02-03T17:31:38.924Z 27 Acked 1026 Read 5 Done Done Sent false
205302024-02-03T17:31:38.924Z 28 Acked 1027 Write 2 Done Done Sent false
205312024-02-03T17:31:38.924Z 29 Acked 1028 Read 4 Done Done Sent false
205322024-02-03T17:31:38.925Z 30 Acked 1029 Write 3 Sent Sent Sent false
205332024-02-03T17:31:38.925Z STATES DS:0 DS:1 DS:2 TOTAL
205342024-02-03T17:31:38.925Z New 0 0 0 0
205352024-02-03T17:31:38.925Z Sent 1 1 28 30
205362024-02-03T17:31:38.925Z Done 27 27 0 54
205372024-02-03T17:31:38.925Z Skipped 0 0 0 0
205382024-02-03T17:31:38.926Z Error 0 0 0 0
205392024-02-03T17:31:38.926ZLast Flush: 1021 1021 1001
205402024-02-03T17:31:38.926ZDownstairs last five completed: 1001 1000
205412024-02-03T17:31:38.926ZUpstairs last five completed: 30 29 28 27 26
205422024-02-03T17:31:38.926Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:38.075812735Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1895}
205432024-02-03T17:31:38.926ZWrote out file "/tmp/test_repair_verify.data"
205442024-02-03T17:31:38.926Z
205452024-02-03T17:31:38.948ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205462024-02-03T17:31:38.948ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205472024-02-03T17:31:38.949Z 0 000-019 183 183 183 334 334 334 F F F
205482024-02-03T17:31:38.949Z 1 020-039 187 187 187 343 343 343 F F F
205492024-02-03T17:31:38.949Z 2 040-059 180 180 180 329 329 329 T T F <---
205502024-02-03T17:31:38.949Z 3 060-079 190 190 187 352 352 343 F F F <---
205512024-02-03T17:31:38.949Z 4 080-099 187 187 187 343 343 343 T T F <---
205522024-02-03T17:31:38.949Z 5 100-119 190 190 181 351 351 332 F F F <---
205532024-02-03T17:31:38.951Z 6 120-139 187 187 187 343 343 343 F F F
205542024-02-03T17:31:38.951Z 7 140-159 189 189 189 349 349 349 F F F
205552024-02-03T17:31:38.951Z 8 160-179 189 189 189 349 349 349 F F F
205562024-02-03T17:31:38.951Z 9 180-199 190 190 190 350 350 350 F F F
205572024-02-03T17:31:38.951Z 10 200-219 190 190 190 350 350 350 F F F
205582024-02-03T17:31:38.951Z 11 220-239 188 188 188 345 345 345 F F F
205592024-02-03T17:31:38.951Z 12 240-259 188 188 188 345 345 345 T T F <---
205602024-02-03T17:31:38.951Z 13 260-279 185 185 185 338 338 338 F F F
205612024-02-03T17:31:38.952Z 14 280-299 188 188 188 345 345 345 F F F
205622024-02-03T17:31:38.952Z 15 300-319 190 190 185 351 351 338 F F F <---
205632024-02-03T17:31:38.952Z 16 320-339 190 190 188 351 351 345 F F F <---
205642024-02-03T17:31:38.952Z 17 340-359 190 190 180 354 354 329 F F F <---
205652024-02-03T17:31:38.952Z 18 360-379 190 190 186 354 354 340 F F F <---
205662024-02-03T17:31:38.952Z 19 380-399 184 184 184 337 337 337 F F F
205672024-02-03T17:31:38.952Z 20 400-419 189 189 189 349 349 349 F F F
205682024-02-03T17:31:38.953Z 21 420-439 190 190 188 354 354 346 F F F <---
205692024-02-03T17:31:38.953Z 22 440-459 188 188 188 346 346 346 F F F
205702024-02-03T17:31:38.953Z 23 460-479 189 189 189 349 349 349 F F F
205712024-02-03T17:31:38.953Z 24 480-499 190 190 186 353 353 342 F F F <---
205722024-02-03T17:31:38.953Z 25 500-519 187 187 187 343 343 343 F F F
205732024-02-03T17:31:38.953Z 26 520-539 184 184 184 337 337 337 F F F
205742024-02-03T17:31:38.954Z 27 540-559 186 186 186 339 339 339 T T F <---
205752024-02-03T17:31:38.954Z 28 560-579 190 190 186 354 354 340 T T F <---
205762024-02-03T17:31:38.954Z 29 580-599 190 190 184 354 354 336 T T F <---
205772024-02-03T17:31:38.954ZMax gen: 190, Max flush: 354
205782024-02-03T17:31:38.954ZError: Difference in extent metadata found!
205792024-02-03T17:31:38.961ZA Difference in extent metadata is expected here
205802024-02-03T17:31:38.961ZOn loop 95
205812024-02-03T17:31:38.962Z
205822024-02-03T17:31:38.962ZVerifying data now
205832024-02-03T17:31:44.159ZNo change after verify
205842024-02-03T17:31:44.160ZLoop: 95 Downstairs dump after verify (and repair):
205852024-02-03T17:31:44.185ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205862024-02-03T17:31:44.185Z 0 000-019 183 183 183 334 334 334 F F F
205872024-02-03T17:31:44.190Z 1 020-039 187 187 187 343 343 343 F F F
205882024-02-03T17:31:44.190Z 2 040-059 191 191 191 355 355 355 F F F
205892024-02-03T17:31:44.191Z 3 060-079 190 190 190 352 352 352 F F F
205902024-02-03T17:31:44.191Z 4 080-099 191 191 191 355 355 355 F F F
205912024-02-03T17:31:44.191Z 5 100-119 190 190 190 351 351 351 F F F
205922024-02-03T17:31:44.191Z 6 120-139 187 187 187 343 343 343 F F F
205932024-02-03T17:31:44.191Z 7 140-159 189 189 189 349 349 349 F F F
205942024-02-03T17:31:44.191Z 8 160-179 189 189 189 349 349 349 F F F
205952024-02-03T17:31:44.191Z 9 180-199 190 190 190 350 350 350 F F F
205962024-02-03T17:31:44.191Z 10 200-219 190 190 190 350 350 350 F F F
205972024-02-03T17:31:44.191Z 11 220-239 188 188 188 345 345 345 F F F
205982024-02-03T17:31:44.191Z 12 240-259 191 191 191 355 355 355 F F F
205992024-02-03T17:31:44.191Z 13 260-279 185 185 185 338 338 338 F F F
206002024-02-03T17:31:44.191Z 14 280-299 188 188 188 345 345 345 F F F
206012024-02-03T17:31:44.191Z 15 300-319 190 190 190 351 351 351 F F F
206022024-02-03T17:31:44.191Z 16 320-339 190 190 190 351 351 351 F F F
206032024-02-03T17:31:44.192Z 17 340-359 190 190 190 354 354 354 F F F
206042024-02-03T17:31:44.192Z 18 360-379 190 190 190 354 354 354 F F F
206052024-02-03T17:31:44.192Z 19 380-399 184 184 184 337 337 337 F F F
206062024-02-03T17:31:44.192Z 20 400-419 189 189 189 349 349 349 F F F
206072024-02-03T17:31:44.192Z 21 420-439 190 190 190 354 354 354 F F F
206082024-02-03T17:31:44.192Z 22 440-459 188 188 188 346 346 346 F F F
206092024-02-03T17:31:44.192Z 23 460-479 189 189 189 349 349 349 F F F
206102024-02-03T17:31:44.192Z 24 480-499 190 190 190 353 353 353 F F F
206112024-02-03T17:31:44.192Z 25 500-519 187 187 187 343 343 343 F F F
206122024-02-03T17:31:44.192Z 26 520-539 184 184 184 337 337 337 F F F
206132024-02-03T17:31:44.192Z 27 540-559 191 191 191 355 355 355 F F F
206142024-02-03T17:31:44.192Z 28 560-579 191 191 191 355 355 355 F F F
206152024-02-03T17:31:44.192Z 29 580-599 191 191 191 355 355 355 F F F
206162024-02-03T17:31:44.193ZMax gen: 191, Max flush: 355
206172024-02-03T17:31:44.193Z
206182024-02-03T17:31:44.193ZBegin loop 96
206192024-02-03T17:31:44.193ZDownstairs to restart: 2
206202024-02-03T17:31:44.223Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.400114517Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903}
206212024-02-03T17:31:44.223Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.400613265Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903}
206222024-02-03T17:31:44.223Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.400646823Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903}
206232024-02-03T17:31:44.241Z{"msg":"Crucible baadf9ef-acdc-4fbc-90e2-ce41b5f727bd has session id: 9d140884-56bd-444e-b1aa-b78f9326afc7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.40071458Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206242024-02-03T17:31:44.241Z{"msg":"Crucible runtime is spawned
206252024-02-03T17:31:44.241ZUpstairs opts: Upstairs UUID: baadf9ef-acdc-4fbc-90e2-ce41b5f727bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.400903459Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206262024-02-03T17:31:44.241Z{"msg":"Crucible stats registered with UUID: baadf9ef-acdc-4fbc-90e2-ce41b5f727bd","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.400949725Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206272024-02-03T17:31:44.241Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.40098946Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903}
206282024-02-03T17:31:44.241Z{"msg":"baadf9ef-acdc-4fbc-90e2-ce41b5f727bd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401169792Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206292024-02-03T17:31:44.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401232195Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206302024-02-03T17:31:44.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401276428Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206312024-02-03T17:31:44.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401332387Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206322024-02-03T17:31:44.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401375333Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206332024-02-03T17:31:44.241Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401415163Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206342024-02-03T17:31:44.241Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401454816Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206352024-02-03T17:31:44.241Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401489984Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206362024-02-03T17:31:44.241Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401560502Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206372024-02-03T17:31:44.241Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401791381Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206382024-02-03T17:31:44.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401896102Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206392024-02-03T17:31:44.241Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.401944366Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206402024-02-03T17:31:44.242Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.402015906Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206412024-02-03T17:31:44.242Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.403132404Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206422024-02-03T17:31:44.242Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.403218179Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206432024-02-03T17:31:44.242Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.40326658Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206442024-02-03T17:31:44.242Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.403337489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206452024-02-03T17:31:44.242Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.40407554Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206462024-02-03T17:31:44.242Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.404146817Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206472024-02-03T17:31:44.242Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.404181578Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206482024-02-03T17:31:44.242Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.404346739Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206492024-02-03T17:31:44.242Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.405122525Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206502024-02-03T17:31:44.242Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.405185321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206512024-02-03T17:31:44.242Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.405210779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206522024-02-03T17:31:44.242Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.405255687Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206532024-02-03T17:31:44.242Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.405279668Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206542024-02-03T17:31:44.242Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.405321823Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206552024-02-03T17:31:44.242Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.405345831Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206562024-02-03T17:31:44.242Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406098115Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206572024-02-03T17:31:44.242Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406162645Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206582024-02-03T17:31:44.242Z{"msg":"[0]R flush_numbers[0..12]: [334, 343, 355, 352, 355, 351, 343, 349, 349, 350, 350, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406208961Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206592024-02-03T17:31:44.242Z{"msg":"[0]R generation[0..12]: [183, 187, 191, 190, 191, 190, 187, 189, 189, 190, 190, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406267816Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206602024-02-03T17:31:44.247Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406313007Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206612024-02-03T17:31:44.247Z{"msg":"[1]R flush_numbers[0..12]: [334, 343, 355, 352, 355, 351, 343, 349, 349, 350, 350, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406372153Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206622024-02-03T17:31:44.247Z{"msg":"[1]R generation[0..12]: [183, 187, 191, 190, 191, 190, 187, 189, 189, 190, 190, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406414303Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206632024-02-03T17:31:44.247Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406454774Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206642024-02-03T17:31:44.247Z{"msg":"[2]R flush_numbers[0..12]: [334, 343, 355, 352, 355, 351, 343, 349, 349, 350, 350, 345]Wait for a query_work_queue command to finish before sending IO
206652024-02-03T17:31:44.247Z","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406544338Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206662024-02-03T17:31:44.247Z{"msg":"[2]R generation[0..12]: [183, 187, 191, 190, 191, 190, 187, 189, 189, 190, 190, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.40660672Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206672024-02-03T17:31:44.247Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406652045Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206682024-02-03T17:31:44.247Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406708295Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206692024-02-03T17:31:44.247Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406750209Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206702024-02-03T17:31:44.248Z{"msg":"Next flush: 356","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.40680609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206712024-02-03T17:31:44.248Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406847981Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206722024-02-03T17:31:44.248Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.40690447Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206732024-02-03T17:31:44.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.406942777Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206742024-02-03T17:31:44.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.407001802Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206752024-02-03T17:31:44.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.407044113Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206762024-02-03T17:31:44.248Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.407102609Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206772024-02-03T17:31:44.248Zpid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206782024-02-03T17:31:44.248Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.407131251Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206792024-02-03T17:31:44.248Z{"msg":"baadf9ef-acdc-4fbc-90e2-ce41b5f727bd is now active with session: 9d140884-56bd-444e-b1aa-b78f9326afc7","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.407172109Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206802024-02-03T17:31:44.248Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.407194173Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903,"session_id":"9d140884-56bd-444e-b1aa-b78f9326afc7"}
206812024-02-03T17:31:44.248Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:43.407233466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903}
206822024-02-03T17:31:44.248ZLoading write count information from file "/tmp/test_repair_verify.data"
206832024-02-03T17:31:44.248ZRun Repair workload
206842024-02-03T17:31:44.248Z01/30 Flush
206852024-02-03T17:31:44.248Z02/30 Write block 208 len 12288 data: 13 11 14
206862024-02-03T17:31:44.248Z03/30 Write block 71 len 40960 data: 9 8 10 11 11 13 13 12 12 13
206872024-02-03T17:31:44.248Z04/30 Write block 556 len 12288 data: 18 16 18
206882024-02-03T17:31:44.248Z05/30 Read block 454 len 32768
206892024-02-03T17:31:44.248Z06/30 Read block 436 len 8192
206902024-02-03T17:31:44.248Z07/30 Read block 454 len 24576
206912024-02-03T17:31:44.248Z08/30 Read block 467 len 24576
206922024-02-03T17:31:44.248Z09/30 Read block 335 len 4096
206932024-02-03T17:31:44.248Z10/30 Read block 11 len 8192
206942024-02-03T17:31:44.249Z11/30 Write block 193 len 12288 data: 9 8 12
206952024-02-03T17:31:44.249Z12/30 Flush
206962024-02-03T17:31:44.261Z13/30 Flush
206972024-02-03T17:31:44.261Z14/30 Write block 114 len 28672 data: 20 19 19 18 17 17 16
206982024-02-03T17:31:44.261Z15/30 Read block 429 len 36864
206992024-02-03T17:31:44.261Z16/30 Write block 386 len 4096 data: 10
207002024-02-03T17:31:44.261Z17/30 Read block 488 len 20480
207012024-02-03T17:31:44.269Z18/30 Write block 309 len 4096 data: 16
207022024-02-03T17:31:44.269Z19/30 Write block 305 len 12288 data: 15 17 13
207032024-02-03T17:31:44.269Z20/30 Read block 54 len 40960
207042024-02-03T17:31:44.270Z21/30 Flush
207052024-02-03T17:31:44.284Z22/30 Write block 561 len 32768 data: 14 10 11 12 15 17 17 14
207062024-02-03T17:31:44.284Z23/30 Flush
207072024-02-03T17:31:44.284Z24/30 Write block 209 len 32768 data: 12 15 13 12 12 13 15 12
207082024-02-03T17:31:44.285Z25/30 Write block 94 len 24576 data: 13 14 15 15 12 14
207092024-02-03T17:31:44.703Z26/30 Read block 475 len 20480
207102024-02-03T17:31:44.703Z27/30 Write block 582 len 4096 data: 22
207112024-02-03T17:31:44.704Z28/30 Read block 385 len 8192
207122024-02-03T17:31:44.704Z29/30 Read block 568 len 32768
207132024-02-03T17:31:44.704Z30/30 Write block 403 len 36864 data: 15 13 18 15 17 15 17 15 16
207142024-02-03T17:31:44.704Z----------------------------------------------------------------
207152024-02-03T17:31:44.704Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
207162024-02-03T17:31:44.704ZStates: Active Active Active
207172024-02-03T17:31:44.705ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
207182024-02-03T17:31:44.705Z 1 Acked 1000 Flush 0 Done Done Sent false
207192024-02-03T17:31:44.705Z 2 Acked 1001 Write 3 Done Done Sent false
207202024-02-03T17:31:44.705Z 3 Acked 1002 Write 10 Done Done Sent false
207212024-02-03T17:31:44.705Z 4 Acked 1003 Write 3 Done Done Sent false
207222024-02-03T17:31:44.706Z 5 Acked 1004 Read 8 Done Done Sent false
207232024-02-03T17:31:44.706Z 6 Acked 1005 Read 2 Done Done Sent false
207242024-02-03T17:31:44.706Z 7 Acked 1006 Read 6 Done Done Sent false
207252024-02-03T17:31:44.706Z 8 Acked 1007 Read 6 Done Done Sent false
207262024-02-03T17:31:44.706Z 9 Acked 1008 Read 1 Done Done Sent false
207272024-02-03T17:31:44.706Z 10 Acked 1009 Read 2 Done Done Sent false
207282024-02-03T17:31:44.707Z 11 Acked 1010 Write 3 Done Done Sent false
207292024-02-03T17:31:44.707Z 12 Acked 1011 Flush 0 Done Done Sent false
207302024-02-03T17:31:44.707Z 13 Acked 1012 Flush 0 Done Done Sent false
207312024-02-03T17:31:44.707Z 14 Acked 1013 Write 7 Done Done Sent false
207322024-02-03T17:31:44.707Z 15 Acked 1014 Read 9 Done Done Sent false
207332024-02-03T17:31:44.707Z 16 Acked 1015 Write 1 Done Done Sent false
207342024-02-03T17:31:44.708Z 17 Acked 1016 Read 5 Done Done Sent false
207352024-02-03T17:31:44.708Z 18 Acked 1017 Write 1 Done Done Sent false
207362024-02-03T17:31:44.708Z 19 Acked 1018 Write 3 Done Done Sent false
207372024-02-03T17:31:44.708Z 20 Acked 1019 Read 10 Done Done Sent false
207382024-02-03T17:31:44.708Z 21 Acked 1020 Flush 0 Done Done Sent false
207392024-02-03T17:31:44.708Z 22 Acked 1021 Write 8 Done Done Sent false
207402024-02-03T17:31:44.709Z 23 Acked 1022 Flush 0 Done Done Sent false
207412024-02-03T17:31:44.709Z 24 Acked 1023 Write 8 Done Done Sent false
207422024-02-03T17:31:44.709Z 25 Acked 1024 Write 6 Done Done Sent false
207432024-02-03T17:31:44.709Z 26 Acked 1025 Read 5 Done Done Sent false
207442024-02-03T17:31:44.709Z 27 Acked 1026 Write 1 Done Done Sent false
207452024-02-03T17:31:44.709Z 28 Acked 1027 Read 2 Done Done Sent false
207462024-02-03T17:31:44.710Z 29 Acked 1028 Read 8 Done Done Sent false
207472024-02-03T17:31:44.710Z 30 Acked 1029 Write 9 Sent Sent Sent false
207482024-02-03T17:31:44.710Z STATES DS:0 DS:1 DS:2 TOTAL
207492024-02-03T17:31:44.710Z New 0 0 0 0
207502024-02-03T17:31:44.710Z Sent 1 1 30 32
207512024-02-03T17:31:44.710Z Done 29 29 0 58
207522024-02-03T17:31:44.711Z Skipped 0 0 0 0
207532024-02-03T17:31:44.711Z Error 0 0 0 0
207542024-02-03T17:31:44.711ZLast Flush: 1022 1022 0
207552024-02-03T17:31:44.711ZDownstairs last five completed:
207562024-02-03T17:31:44.711ZUpstairs last five completed: 30 29 28 27 26
207572024-02-03T17:31:44.711Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:43.467703009Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1903}
207582024-02-03T17:31:44.711ZWrote out file "/tmp/test_repair_verify.data"
207592024-02-03T17:31:44.725Z
207602024-02-03T17:31:44.725ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207612024-02-03T17:31:44.725ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207622024-02-03T17:31:44.725Z 0 000-019 183 183 183 334 334 334 F F F
207632024-02-03T17:31:44.725Z 1 020-039 187 187 187 343 343 343 F F F
207642024-02-03T17:31:44.725Z 2 040-059 191 191 191 355 355 355 F F F
207652024-02-03T17:31:44.726Z 3 060-079 192 192 190 357 357 352 F F F <---
207662024-02-03T17:31:44.726Z 4 080-099 192 192 191 357 357 355 T T F <---
207672024-02-03T17:31:44.726Z 5 100-119 192 192 190 359 359 351 F F F <---
207682024-02-03T17:31:44.726Z 6 120-139 192 192 187 359 359 343 F F F <---
207692024-02-03T17:31:44.726Z 7 140-159 189 189 189 349 349 349 F F F
207702024-02-03T17:31:44.727Z 8 160-179 189 189 189 349 349 349 F F F
207712024-02-03T17:31:44.727Z 9 180-199 192 192 190 357 357 350 F F F <---
207722024-02-03T17:31:44.727Z 10 200-219 192 192 190 357 357 350 T T F <---
207732024-02-03T17:31:44.727Z 11 220-239 188 188 188 345 345 345 F F F
207742024-02-03T17:31:44.727Z 12 240-259 191 191 191 355 355 355 F F F
207752024-02-03T17:31:44.727Z 13 260-279 185 185 185 338 338 338 F F F
207762024-02-03T17:31:44.727Z 14 280-299 188 188 188 345 345 345 F F F
207772024-02-03T17:31:44.728Z 15 300-319 192 192 190 359 359 351 F F F <---
207782024-02-03T17:31:44.728Z 16 320-339 190 190 190 351 351 351 F F F
207792024-02-03T17:31:44.728Z 17 340-359 190 190 190 354 354 354 F F F
207802024-02-03T17:31:44.728Z 18 360-379 190 190 190 354 354 354 F F F
207812024-02-03T17:31:44.728Z 19 380-399 192 192 184 359 359 337 F F F <---
207822024-02-03T17:31:44.729Z 20 400-419 189 189 189 349 349 349 T T F <---
207832024-02-03T17:31:44.729Z 21 420-439 190 190 190 354 354 354 F F F
207842024-02-03T17:31:44.729Z 22 440-459 188 188 188 346 346 346 F F F
207852024-02-03T17:31:44.729Z 23 460-479 189 189 189 349 349 349 F F F
207862024-02-03T17:31:44.729Z 24 480-499 190 190 190 353 353 353 F F F
207872024-02-03T17:31:44.730Z 25 500-519 187 187 187 343 343 343 F F F
207882024-02-03T17:31:44.730Z 26 520-539 184 184 184 337 337 337 F F F
207892024-02-03T17:31:44.730Z 27 540-559 192 192 191 357 357 355 F F F <---
207902024-02-03T17:31:44.730Z 28 560-579 192 192 191 360 360 355 F F F <---
207912024-02-03T17:31:44.730Z 29 580-599 191 191 191 355 355 355 T T F <---
207922024-02-03T17:31:44.730ZMax gen: 192, Max flush: 360
207932024-02-03T17:31:44.731ZError: Difference in extent metadata found!
207942024-02-03T17:31:44.731ZA Difference in extent metadata is expected here
207952024-02-03T17:31:44.731ZOn loop 96
207962024-02-03T17:31:44.731Z
207972024-02-03T17:31:44.732ZVerifying data now
207982024-02-03T17:31:49.413ZNo change after verify
207992024-02-03T17:31:49.413ZLoop: 96 Downstairs dump after verify (and repair):
208002024-02-03T17:31:49.444ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208012024-02-03T17:31:49.444Z 0 000-019 183 183 183 334 334 334 F F F
208022024-02-03T17:31:49.451Z 1 020-039 187 187 187 343 343 343 F F F
208032024-02-03T17:31:49.451Z 2 040-059 191 191 191 355 355 355 F F F
208042024-02-03T17:31:49.451Z 3 060-079 192 192 192 357 357 357 F F F
208052024-02-03T17:31:49.451Z 4 080-099 193 193 193 361 361 361 F F F
208062024-02-03T17:31:49.451Z 5 100-119 192 192 192 359 359 359 F F F
208072024-02-03T17:31:49.451Z 6 120-139 192 192 192 359 359 359 F F F
208082024-02-03T17:31:49.451Z 7 140-159 189 189 189 349 349 349 F F F
208092024-02-03T17:31:49.451Z 8 160-179 189 189 189 349 349 349 F F F
208102024-02-03T17:31:49.451Z 9 180-199 192 192 192 357 357 357 F F F
208112024-02-03T17:31:49.451Z 10 200-219 193 193 193 361 361 361 F F F
208122024-02-03T17:31:49.451Z 11 220-239 188 188 188 345 345 345 F F F
208132024-02-03T17:31:49.451Z 12 240-259 191 191 191 355 355 355 F F F
208142024-02-03T17:31:49.451Z 13 260-279 185 185 185 338 338 338 F F F
208152024-02-03T17:31:49.451Z 14 280-299 188 188 188 345 345 345 F F F
208162024-02-03T17:31:49.451Z 15 300-319 192 192 192 359 359 359 F F F
208172024-02-03T17:31:49.452Z 16 320-339 190 190 190 351 351 351 F F F
208182024-02-03T17:31:49.452Z 17 340-359 190 190 190 354 354 354 F F F
208192024-02-03T17:31:49.452Z 18 360-379 190 190 190 354 354 354 F F F
208202024-02-03T17:31:49.452Z 19 380-399 192 192 192 359 359 359 F F F
208212024-02-03T17:31:49.452Z 20 400-419 193 193 193 361 361 361 F F F
208222024-02-03T17:31:49.452Z 21 420-439 190 190 190 354 354 354 F F F
208232024-02-03T17:31:49.452Z 22 440-459 188 188 188 346 346 346 F F F
208242024-02-03T17:31:49.452Z 23 460-479 189 189 189 349 349 349 F F F
208252024-02-03T17:31:49.452Z 24 480-499 190 190 190 353 353 353 F F F
208262024-02-03T17:31:49.452Z 25 500-519 187 187 187 343 343 343 F F F
208272024-02-03T17:31:49.452Z 26 520-539 184 184 184 337 337 337 F F F
208282024-02-03T17:31:49.452Z 27 540-559 192 192 192 357 357 357 F F F
208292024-02-03T17:31:49.452Z 28 560-579 192 192 192 360 360 360 F F F
208302024-02-03T17:31:49.452Z 29 580-599 193 193 193 361 361 361 F F F
208312024-02-03T17:31:49.452ZMax gen: 193, Max flush: 361
208322024-02-03T17:31:49.452Z
208332024-02-03T17:31:49.452ZBegin loop 97
208342024-02-03T17:31:49.452ZDownstairs to restart: 0
208352024-02-03T17:31:49.492Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.654956829Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911}
208362024-02-03T17:31:49.492Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.65540182Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911}
208372024-02-03T17:31:49.492Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.655432097Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911}
208382024-02-03T17:31:49.519Z{"msg":"Crucible e8401cb3-9a46-4ef9-83da-0cb943e9753c has session id: 16a015e2-c431-4061-9281-17e5afc53340","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.660184749Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208392024-02-03T17:31:49.519Z{"msg":"Upstairs opts: Upstairs UUID: e8401cb3-9a46-4ef9-83da-0cb943e9753c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.660258472Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208402024-02-03T17:31:49.519Z{"msg":"Crucible stats registered with UUID: e8401cb3-9a46-4ef9-83da-0cb943e9753c","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.660285782Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208412024-02-03T17:31:49.519ZCrucible runtime is spawned
208422024-02-03T17:31:49.519Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.660856366Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911}
208432024-02-03T17:31:49.519Z{"msg":"e8401cb3-9a46-4ef9-83da-0cb943e9753c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.66103039Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208442024-02-03T17:31:49.519Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661092969Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208452024-02-03T17:31:49.519Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661121776Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208462024-02-03T17:31:49.519Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661190594Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208472024-02-03T17:31:49.519Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661223385Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208482024-02-03T17:31:49.519Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661247246Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208492024-02-03T17:31:49.520Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661312794Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208502024-02-03T17:31:49.520Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661354952Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208512024-02-03T17:31:49.520Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661396384Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208522024-02-03T17:31:49.520Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661438222Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208532024-02-03T17:31:49.520Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661483728Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208542024-02-03T17:31:49.520Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.661524199Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208552024-02-03T17:31:49.520Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.662040611Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208562024-02-03T17:31:49.520Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.66313095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208572024-02-03T17:31:49.520Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.663187112Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208582024-02-03T17:31:49.520Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.66321158Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208592024-02-03T17:31:49.520Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.663286219Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208602024-02-03T17:31:49.520Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.663311003Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208612024-02-03T17:31:49.521Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.663332857Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208622024-02-03T17:31:49.521Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.664126653Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208632024-02-03T17:31:49.521Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.664199814Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208642024-02-03T17:31:49.521Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.664256919Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208652024-02-03T17:31:49.521Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665306675Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208662024-02-03T17:31:49.521Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665384673Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208672024-02-03T17:31:49.521Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665412499Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208682024-02-03T17:31:49.521Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665453901Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208692024-02-03T17:31:49.521Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665478236Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208702024-02-03T17:31:49.521Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665519298Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208712024-02-03T17:31:49.521Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665541888Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208722024-02-03T17:31:49.521Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665583169Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208732024-02-03T17:31:49.521Z{"msg":"[0]R flush_numbers[0..12]: [334, 343, 355, 357, 361, 359, 359, 349, 349, 357, 361, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665608862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208742024-02-03T17:31:49.521Z{"msg":"[0]R generation[0..12]: [183, 187, 191, 192, 193, 192, 192, 189, 189, 192, 193, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665650937Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208752024-02-03T17:31:49.522Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.66567676Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208762024-02-03T17:31:49.522Z{"msg":"[1]R flush_numbers[0..12]: [334, 343, 355, 357, 361, 359, 359, 349, 349, 357, 361, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665719346Z","Wait for a query_work_queue command to finish before sending IO
208772024-02-03T17:31:49.522Zhostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208782024-02-03T17:31:49.522Z{"msg":"[1]R generation[0..12]: [183, 187, 191, 192, 193, 192, 192, 189, 189, 192, 193, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665865844Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208792024-02-03T17:31:49.522Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665896156Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208802024-02-03T17:31:49.522Z{"msg":"[2]R flush_numbers[0..12]: [334, 343, 355, 357, 361, 359, 359, 349, 349, 357, 361, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665939907Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208812024-02-03T17:31:49.522Z{"msg":"[2]R generation[0..12]: [183, 187, 191, 192, 193, 192, 192, 189, 189, 192, 193, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.665964473Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208822024-02-03T17:31:49.522Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666007432Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208832024-02-03T17:31:49.522Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.66603098Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208842024-02-03T17:31:49.522Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666053651Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208852024-02-03T17:31:49.522Z{"msg":"Next flush: 362","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666112681Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208862024-02-03T17:31:49.522Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666150452Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208872024-02-03T17:31:49.522Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666216637Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208882024-02-03T17:31:49.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666239191Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208892024-02-03T17:31:49.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666260848Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208902024-02-03T17:31:49.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.66630317Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208912024-02-03T17:31:49.523Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666326673Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208922024-02-03T17:31:49.523Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666366607Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208932024-02-03T17:31:49.523Z{"msg":"e8401cb3-9a46-4ef9-83da-0cb943e9753c is now active with session: 16a015e2-c431-4061-9281-17e5afc53340","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666389614Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208942024-02-03T17:31:49.523Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666435832Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911,"session_id":"16a015e2-c431-4061-9281-17e5afc53340"}
208952024-02-03T17:31:49.523Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:48.666457556Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911}
208962024-02-03T17:31:49.523ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208972024-02-03T17:31:49.523ZLoading write count information from file "/tmp/test_repair_verify.data"
208982024-02-03T17:31:49.523ZRun Repair workload
208992024-02-03T17:31:49.523Z01/30 Write block 256 len 24576 data: 15 13 11 11 15 16
209002024-02-03T17:31:49.523Z02/30 Flush
209012024-02-03T17:31:49.524Z03/30 Write block 465 len 28672 data: 18 15 15 14 15 15 17
209022024-02-03T17:31:49.524Z04/30 Flush
209032024-02-03T17:31:49.524Z05/30 Write block 84 len 12288 data: 17 12 12
209042024-02-03T17:31:49.524Z06/30 Read block 48 len 20480
209052024-02-03T17:31:49.528Z07/30 Read block 90 len 28672
209062024-02-03T17:31:49.528Z08/30 Read block 342 len 8192
209072024-02-03T17:31:49.528Z09/30 Read block 36 len 24576
209082024-02-03T17:31:49.528Z10/30 Read block 512 len 40960
209092024-02-03T17:31:49.528Z11/30 Write block 94 len 12288 data: 14 15 16
209102024-02-03T17:31:49.528Z12/30 Read block 561 len 20480
209112024-02-03T17:31:49.528Z13/30 Read block 447 len 40960
209122024-02-03T17:31:49.528Z14/30 Read block 85 len 28672
209132024-02-03T17:31:49.528Z15/30 Read block 54 len 36864
209142024-02-03T17:31:49.528Z16/30 Write block 182 len 40960 data: 15 13 10 11 11 12 12 13 12 12
209152024-02-03T17:31:49.529Z17/30 Write block 78 len 8192 data: 13 13
209162024-02-03T17:31:49.529Z18/30 Read block 38 len 20480
209172024-02-03T17:31:49.529Z19/30 Write block 269 len 40960 data: 20 20 19 18 18 21 22 21 20 20
209182024-02-03T17:31:49.529Z20/30 Write block 317 len 4096 data: 10
209192024-02-03T17:31:49.529Z21/30 Read block 388 len 20480
209202024-02-03T17:31:49.529Z22/30 Flush
209212024-02-03T17:31:50.539Z23/30 Write block 201 len 36864 data: 12 14 15 15 12 10 11 14 13
209222024-02-03T17:31:50.539Z24/30 Flush
209232024-02-03T17:31:50.547Z25/30 Write block 211 len 20480 data: 14 13 13 14 16
209242024-02-03T17:31:50.547Z26/30 Read block 493 len 40960
209252024-02-03T17:31:50.547Z27/30 Read block 415 len 24576
209262024-02-03T17:31:50.548Z28/30 Write block 280 len 36864 data: 16 18 20 18 17 14 13 14 12
209272024-02-03T17:31:50.548Z29/30 Write block 91 len 24576 data: 11 12 13 15 16 17
209282024-02-03T17:31:50.548Z30/30 Read block 177 len 8192
209292024-02-03T17:31:50.548Z----------------------------------------------------------------
209302024-02-03T17:31:50.548Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:28
209312024-02-03T17:31:50.549ZStates: Active Active Active
209322024-02-03T17:31:50.549ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209332024-02-03T17:31:50.549Z 3 Acked 1002 Write 7 Sent Done Done false
209342024-02-03T17:31:50.549Z 4 Acked 1003 Flush 0 Sent Done Done false
209352024-02-03T17:31:50.549Z 5 Acked 1004 Write 3 Sent Done Done false
209362024-02-03T17:31:50.550Z 6 Acked 1005 Read 5 Sent Done Done false
209372024-02-03T17:31:50.550Z 7 Acked 1006 Read 7 Sent Done Done false
209382024-02-03T17:31:50.550Z 8 Acked 1007 Read 2 Sent Done Done false
209392024-02-03T17:31:50.550Z 9 Acked 1008 Read 6 Sent Done Done false
209402024-02-03T17:31:50.551Z 10 Acked 1009 Read 10 Sent Done Done false
209412024-02-03T17:31:50.551Z 11 Acked 1010 Write 3 Sent Done Done false
209422024-02-03T17:31:50.551Z 12 Acked 1011 Read 5 Sent Done Done false
209432024-02-03T17:31:50.552Z 13 Acked 1012 Read 10 Sent Done Done false
209442024-02-03T17:31:50.552Z 14 Acked 1013 Read 7 Sent Done Done false
209452024-02-03T17:31:50.552Z 15 Acked 1014 Read 9 Sent Done Done false
209462024-02-03T17:31:50.552Z 16 Acked 1015 Write 10 Sent Done Done false
209472024-02-03T17:31:50.552Z 17 Acked 1016 Write 2 Sent Done Done false
209482024-02-03T17:31:50.552Z 18 Acked 1017 Read 5 Sent Done Done false
209492024-02-03T17:31:50.553Z 19 Acked 1018 Write 10 Sent Done Done false
209502024-02-03T17:31:50.553Z 20 Acked 1019 Write 1 Sent Done Done false
209512024-02-03T17:31:50.553Z 21 Acked 1020 Read 5 Sent Done Done false
209522024-02-03T17:31:50.553Z 22 Acked 1021 Flush 0 Sent Done Done false
209532024-02-03T17:31:50.553Z 23 Acked 1022 Write 9 Sent Done Done false
209542024-02-03T17:31:50.554Z 24 Acked 1023 Flush 0 Sent Done Done false
209552024-02-03T17:31:50.554Z 25 Acked 1024 Write 5 Sent Done Done false
209562024-02-03T17:31:50.554Z 26 Acked 1025 Read 10 Sent Done Done false
209572024-02-03T17:31:50.554Z 27 Acked 1026 Read 6 Sent Done Done false
209582024-02-03T17:31:50.554Z 28 Acked 1027 Write 9 Sent Done Done false
209592024-02-03T17:31:50.554Z 29 Acked 1028 Write 6 Sent Done Done false
209602024-02-03T17:31:50.555Z 30 Acked 1029 Read 2 Sent Done Done false
209612024-02-03T17:31:50.555Z STATES DS:0 DS:1 DS:2 TOTAL
209622024-02-03T17:31:50.555Z New 0 0 0 0
209632024-02-03T17:31:50.555Z Sent 28 0 0 28
209642024-02-03T17:31:50.555Z Done 0 28 28 56
209652024-02-03T17:31:50.555Z Skipped 0 0 0 0
209662024-02-03T17:31:50.555Z Error 0 0 0 0
209672024-02-03T17:31:50.556ZLast Flush: 1001 1023 1023
209682024-02-03T17:31:50.556ZDownstairs last five completed: 1001 1000
209692024-02-03T17:31:50.556ZUpstairs last five completed: 30 29 28 27 26
209702024-02-03T17:31:50.556Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:48.721034917Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1911}
209712024-02-03T17:31:50.556ZWrote out file "/tmp/test_repair_verify.data"
209722024-02-03T17:31:50.556Z
209732024-02-03T17:31:50.557ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209742024-02-03T17:31:50.557ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209752024-02-03T17:31:50.557Z 0 000-019 183 183 183 334 334 334 F F F
209762024-02-03T17:31:50.557Z 1 020-039 187 187 187 343 343 343 F F F
209772024-02-03T17:31:50.557Z 2 040-059 191 191 191 355 355 355 F F F
209782024-02-03T17:31:50.557Z 3 060-079 192 194 194 357 364 364 F F F <---
209792024-02-03T17:31:50.558Z 4 080-099 193 194 194 361 364 364 F T T <---
209802024-02-03T17:31:50.558Z 5 100-119 192 192 192 359 359 359 F F F
209812024-02-03T17:31:50.558Z 6 120-139 192 192 192 359 359 359 F F F
209822024-02-03T17:31:50.558Z 7 140-159 189 189 189 349 349 349 F F F
209832024-02-03T17:31:50.558Z 8 160-179 189 189 189 349 349 349 F F F
209842024-02-03T17:31:50.558Z 9 180-199 192 194 194 357 364 364 F F F <---
209852024-02-03T17:31:50.559Z 10 200-219 193 194 194 361 365 365 F T T <---
209862024-02-03T17:31:50.559Z 11 220-239 188 188 188 345 345 345 F F F
209872024-02-03T17:31:50.559Z 12 240-259 194 194 194 362 362 362 F F F
209882024-02-03T17:31:50.559Z 13 260-279 194 194 194 362 364 364 F F F <---
209892024-02-03T17:31:50.559Z 14 280-299 188 188 188 345 345 345 F T T <---
209902024-02-03T17:31:50.559Z 15 300-319 192 194 194 359 364 364 F F F <---
209912024-02-03T17:31:50.559Z 16 320-339 190 190 190 351 351 351 F F F
209922024-02-03T17:31:50.559Z 17 340-359 190 190 190 354 354 354 F F F
209932024-02-03T17:31:50.560Z 18 360-379 190 190 190 354 354 354 F F F
209942024-02-03T17:31:50.560Z 19 380-399 192 192 192 359 359 359 F F F
209952024-02-03T17:31:50.560Z 20 400-419 193 193 193 361 361 361 F F F
209962024-02-03T17:31:50.560Z 21 420-439 190 190 190 354 354 354 F F F
209972024-02-03T17:31:50.560Z 22 440-459 188 188 188 346 346 346 F F F
209982024-02-03T17:31:50.561Z 23 460-479 189 194 194 349 363 363 F F F <---
209992024-02-03T17:31:50.561Z 24 480-499 190 190 190 353 353 353 F F F
210002024-02-03T17:31:50.561Z 25 500-519 187 187 187 343 343 343 F F F
210012024-02-03T17:31:50.561Z 26 520-539 184 184 184 337 337 337 F F F
210022024-02-03T17:31:50.561Z 27 540-559 192 192 192 357 357 357 F F F
210032024-02-03T17:31:50.561Z 28 560-579 192 192 192 360 360 360 F F F
210042024-02-03T17:31:50.561Z 29 580-599 193 193 193 361 361 361 F F F
210052024-02-03T17:31:50.562ZMax gen: 194, Max flush: 365
210062024-02-03T17:31:50.562ZError: Difference in extent metadata found!
210072024-02-03T17:31:50.562ZA Difference in extent metadata is expected here
210082024-02-03T17:31:50.562ZOn loop 97
210092024-02-03T17:31:50.562Z
210102024-02-03T17:31:50.562ZVerifying data now
210112024-02-03T17:31:54.381ZNo change after verify
210122024-02-03T17:31:54.382ZLoop: 97 Downstairs dump after verify (and repair):
210132024-02-03T17:31:54.405ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210142024-02-03T17:31:54.405Z 0 000-019 183 183 183 334 334 334 F F F
210152024-02-03T17:31:54.405Z 1 020-039 187 187 187 343 343 343 F F F
210162024-02-03T17:31:54.405Z 2 040-059 191 191 191 355 355 355 F F F
210172024-02-03T17:31:54.409Z 3 060-079 194 194 194 364 364 364 F F F
210182024-02-03T17:31:54.409Z 4 080-099 195 195 195 366 366 366 F F F
210192024-02-03T17:31:54.409Z 5 100-119 192 192 192 359 359 359 F F F
210202024-02-03T17:31:54.409Z 6 120-139 192 192 192 359 359 359 F F F
210212024-02-03T17:31:54.409Z 7 140-159 189 189 189 349 349 349 F F F
210222024-02-03T17:31:54.409Z 8 160-179 189 189 189 349 349 349 F F F
210232024-02-03T17:31:54.409Z 9 180-199 194 194 194 364 364 364 F F F
210242024-02-03T17:31:54.409Z 10 200-219 195 195 195 366 366 366 F F F
210252024-02-03T17:31:54.409Z 11 220-239 188 188 188 345 345 345 F F F
210262024-02-03T17:31:54.409Z 12 240-259 194 194 194 362 362 362 F F F
210272024-02-03T17:31:54.409Z 13 260-279 194 194 194 364 364 364 F F F
210282024-02-03T17:31:54.409Z 14 280-299 195 195 195 366 366 366 F F F
210292024-02-03T17:31:54.409Z 15 300-319 194 194 194 364 364 364 F F F
210302024-02-03T17:31:54.409Z 16 320-339 190 190 190 351 351 351 F F F
210312024-02-03T17:31:54.409Z 17 340-359 190 190 190 354 354 354 F F F
210322024-02-03T17:31:54.409Z 18 360-379 190 190 190 354 354 354 F F F
210332024-02-03T17:31:54.409Z 19 380-399 192 192 192 359 359 359 F F F
210342024-02-03T17:31:54.409Z 20 400-419 193 193 193 361 361 361 F F F
210352024-02-03T17:31:54.409Z 21 420-439 190 190 190 354 354 354 F F F
210362024-02-03T17:31:54.409Z 22 440-459 188 188 188 346 346 346 F F F
210372024-02-03T17:31:54.409Z 23 460-479 194 194 194 363 363 363 F F F
210382024-02-03T17:31:54.410Z 24 480-499 190 190 190 353 353 353 F F F
210392024-02-03T17:31:54.410Z 25 500-519 187 187 187 343 343 343 F F F
210402024-02-03T17:31:54.410Z 26 520-539 184 184 184 337 337 337 F F F
210412024-02-03T17:31:54.410Z 27 540-559 192 192 192 357 357 357 F F F
210422024-02-03T17:31:54.410Z 28 560-579 192 192 192 360 360 360 F F F
210432024-02-03T17:31:54.410Z 29 580-599 193 193 193 361 361 361 F F F
210442024-02-03T17:31:54.410ZMax gen: 195, Max flush: 366
210452024-02-03T17:31:54.413Z
210462024-02-03T17:31:54.413ZBegin loop 98
210472024-02-03T17:31:54.413ZDownstairs to restart: 0
210482024-02-03T17:31:54.444Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.623235538Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919}
210492024-02-03T17:31:54.444Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.623771779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919}
210502024-02-03T17:31:54.444Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.623805421Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919}
210512024-02-03T17:31:54.448Z{"msg":"Crucible 5e367747-860b-4490-92e3-a6b67af6d248 has session id: 05786ecb-a2e7-48d0-9c6c-ffad12aa8986","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.623869002Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210522024-02-03T17:31:54.448Z{"msg":"Upstairs opts: Upstairs UUID: 5e367747-860b-4490-92e3-a6b67af6d248, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.623954957Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210532024-02-03T17:31:54.448Z{"msg":"Crucible stats registered with UUID: 5e367747-860b-4490-92e3-a6b67af6d248","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.624028196Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210542024-02-03T17:31:54.448ZCrucible runtime is spawned
210552024-02-03T17:31:54.448Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.6243334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919}
210562024-02-03T17:31:54.448Z{"msg":"5e367747-860b-4490-92e3-a6b67af6d248 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.624448979Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210572024-02-03T17:31:54.448Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.62450029Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210582024-02-03T17:31:54.448Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.624526449Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210592024-02-03T17:31:54.448Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.62458831Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210602024-02-03T17:31:54.448Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.624654941Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210612024-02-03T17:31:54.448Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.62469529Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210622024-02-03T17:31:54.448Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.624731862Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210632024-02-03T17:31:54.449Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.624777911Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210642024-02-03T17:31:54.449Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.624816561Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210652024-02-03T17:31:54.449Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.624904452Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210662024-02-03T17:31:54.449Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.625151218Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210672024-02-03T17:31:54.449Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.625261686Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210682024-02-03T17:31:54.449Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.625349891Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210692024-02-03T17:31:54.449Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.626497707Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210702024-02-03T17:31:54.449Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.626574944Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210712024-02-03T17:31:54.449Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.626620314Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210722024-02-03T17:31:54.449Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.626740676Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210732024-02-03T17:31:54.449Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.627375439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210742024-02-03T17:31:54.449Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.627477759Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210752024-02-03T17:31:54.454Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.62758866Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210762024-02-03T17:31:54.454Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.627642764Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210772024-02-03T17:31:54.454Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.628322093Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210782024-02-03T17:31:54.454Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.628669061Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210792024-02-03T17:31:54.454Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.628757053Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210802024-02-03T17:31:54.454Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.628832489Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210812024-02-03T17:31:54.454Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.628900254Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210822024-02-03T17:31:54.454Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.628970865Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210832024-02-03T17:31:54.454Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629101739Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210842024-02-03T17:31:54.454Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629299321Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210852024-02-03T17:31:54.454Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629387095Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210862024-02-03T17:31:54.454Z{"msg":"[0]R flush_numbers[0..12]: [334, 343, 355, 364, 366, 359, 359, 349, 349, 364, 366, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629425462Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210872024-02-03T17:31:54.454Z{"msg":"[0]R generation[0..12]: [183, 187, 191, 194, 195, 192, 192, 189, 189, 194, 195, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629487301Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210882024-02-03T17:31:54.454Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629522379Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210892024-02-03T17:31:54.454Z{"msg":"[1]R flush_numbers[0..12]: [334, 343, 355, 364, 366, 359, 359, 349, 349, 364, 366, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629579939Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210902024-02-03T17:31:54.454Z{"msg":"[1]R generation[0..12]: [183, 187, 191, 194, 195, 192, 192, 189, 189, 194, 195, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.62961334Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210912024-02-03T17:31:54.454Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629671924Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
210922024-02-03T17:31:54.454Z05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210932024-02-03T17:31:54.454Z{"msg":"[2]R flush_numbers[0..12]: [334, 343, 355, 364, 366, 359, 359, 349, 349, 364, 366, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629712433Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210942024-02-03T17:31:54.454Z{"msg":"[2]R generation[0..12]: [183, 187, 191, 194, 195, 192, 192, 189, 189, 194, 195, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629766573Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210952024-02-03T17:31:54.454Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629793551Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210962024-02-03T17:31:54.455Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629855103Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210972024-02-03T17:31:54.455Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629882444Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210982024-02-03T17:31:54.455Z{"msg":"Next flush: 367","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629939467Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
210992024-02-03T17:31:54.455Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.629965749Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211002024-02-03T17:31:54.455Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.630023386Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211012024-02-03T17:31:54.455Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.630048754Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211022024-02-03T17:31:54.455Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.630111705Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211032024-02-03T17:31:54.455Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.630138441Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211042024-02-03T17:31:54.455Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.630200555Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211052024-02-03T17:31:54.455Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.630226319Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211062024-02-03T17:31:54.455Z{"msg":"5e367747-860b-4490-92e3-a6b67af6d248 is now active with session: 05786ecb-a2e7-48d0-9c6c-ffad12aa8986","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211072024-02-03T17:31:54.455Z,"level":30,"time":"2024-02-03T17:31:53.630309019Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211082024-02-03T17:31:54.455Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.63036689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919,"session_id":"05786ecb-a2e7-48d0-9c6c-ffad12aa8986"}
211092024-02-03T17:31:54.455Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:31:53.630394951Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919}
211102024-02-03T17:31:54.455ZLoading write count information from file "/tmp/test_repair_verify.data"
211112024-02-03T17:31:54.455ZRun Repair workload
211122024-02-03T17:31:54.455Z01/30 Write block 386 len 20480 data: 11 11 12 13 16
211132024-02-03T17:31:54.459Z02/30 Write block 490 len 12288 data: 18 17 14
211142024-02-03T17:31:54.459Z03/30 Write block 273 len 8192 data: 19 22
211152024-02-03T17:31:54.459Z04/30 Write block 572 len 32768 data: 8 10 11 14 15 17 21 24
211162024-02-03T17:31:54.459Z05/30 Read block 499 len 32768
211172024-02-03T17:31:54.462Z06/30 Read block 580 len 16384
211182024-02-03T17:31:54.463Z07/30 Read block 369 len 28672
211192024-02-03T17:31:54.466Z08/30 Write block 84 len 8192 data: 18 13
211202024-02-03T17:31:54.466Z09/30 Read block 223 len 40960
211212024-02-03T17:31:54.469Z10/30 Write block 319 len 32768 data: 15 12 13 14 13 12 10 12
211222024-02-03T17:31:54.469Z11/30 Read block 235 len 24576
211232024-02-03T17:31:54.469Z12/30 Read block 561 len 12288
211242024-02-03T17:31:54.472Z13/30 Read block 315 len 40960
211252024-02-03T17:31:54.475Z14/30 Write block 390 len 4096 data: 17
211262024-02-03T17:31:54.475Z15/30 Write block 488 len 24576 data: 15 17 19 18 15 16
211272024-02-03T17:31:54.475Z16/30 Write block 18 len 32768 data: 11 10 7 10 12 11 15 14
211282024-02-03T17:31:54.478Z17/30 Read block 587 len 32768
211292024-02-03T17:31:54.478Z18/30 Write block 191 len 24576 data: 13 11 10 9 13 13
211302024-02-03T17:31:54.481Z19/30 Write block 360 len 8192 data: 19 20
211312024-02-03T17:31:54.481Z20/30 Write block 118 len 4096 data: 18
211322024-02-03T17:31:54.481Z21/30 Write block 585 len 20480 data: 19 22 18 19 20
211332024-02-03T17:31:54.481Z22/30 Read block 332 len 40960
211342024-02-03T17:31:54.485Z23/30 Write block 518 len 20480 data: 14 12 10 10 9
211352024-02-03T17:31:54.485Z24/30 Flush
211362024-02-03T17:31:54.491Z25/30 Write block 33 len 32768 data: 9 11 11 11 12 16 16 13
211372024-02-03T17:31:54.494Z26/30 Write block 407 len 32768 data: 18 16 18 16 17 14 16 13
211382024-02-03T17:31:54.494Z27/30 Read block 76 len 8192
211392024-02-03T17:31:54.497Z28/30 Write block 346 len 20480 data: 10 12 14 13 12
211402024-02-03T17:31:54.497Z29/30 Read block 331 len 4096
211412024-02-03T17:31:54.497Z30/30 Read block 160 len 8192
211422024-02-03T17:31:54.501Z----------------------------------------------------------------
211432024-02-03T17:31:54.501Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
211442024-02-03T17:31:54.501ZStates: Active Active Active
211452024-02-03T17:31:54.501ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211462024-02-03T17:31:54.501Z 1 Acked 1000 Write 5 Done Done Done false
211472024-02-03T17:31:54.501Z 2 Acked 1001 Write 3 Done Done Done false
211482024-02-03T17:31:54.502Z 3 Acked 1002 Write 2 Done Done Done false
211492024-02-03T17:31:54.502Z 4 Acked 1003 Write 8 Sent Done Done false
211502024-02-03T17:31:54.502Z 5 Acked 1004 Read 8 Sent Done Done false
211512024-02-03T17:31:54.502Z 6 Acked 1005 Read 4 Sent Done Done false
211522024-02-03T17:31:54.502Z 7 Acked 1006 Read 7 Sent Done Done false
211532024-02-03T17:31:54.502Z 8 Acked 1007 Write 2 Sent Done Done false
211542024-02-03T17:31:54.502Z 9 Acked 1008 Read 10 Sent Done Done false
211552024-02-03T17:31:54.502Z 10 Acked 1009 Write 8 Sent Done Done false
211562024-02-03T17:31:54.502Z 11 Acked 1010 Read 6 Sent Done Done false
211572024-02-03T17:31:54.502Z 12 Acked 1011 Read 3 Sent Done Done false
211582024-02-03T17:31:54.502Z 13 Acked 1012 Read 10 Sent Done Done false
211592024-02-03T17:31:54.502Z 14 Acked 1013 Write 1 Sent Done Done false
211602024-02-03T17:31:54.502Z 15 Acked 1014 Write 6 Sent Done Done false
211612024-02-03T17:31:54.502Z 16 Acked 1015 Write 8 Sent Done Done false
211622024-02-03T17:31:54.502Z 17 Acked 1016 Read 8 Sent Done Done false
211632024-02-03T17:31:54.502Z 18 Acked 1017 Write 6 Sent Done Done false
211642024-02-03T17:31:54.502Z 19 Acked 1018 Write 2 Sent Done Done false
211652024-02-03T17:31:54.502Z 20 Acked 1019 Write 1 Sent Done Done false
211662024-02-03T17:31:54.502Z 21 Acked 1020 Write 5 Sent Done Done false
211672024-02-03T17:31:54.502Z 22 Acked 1021 Read 10 Sent Done Done false
211682024-02-03T17:31:54.502Z 23 Acked 1022 Write 5 Sent Done Done false
211692024-02-03T17:31:54.502Z 24 Acked 1023 Flush 0 Sent Done Done false
211702024-02-03T17:31:54.502Z 25 Acked 1024 Write 8 Sent Done Done false
211712024-02-03T17:31:54.502Z 26 Acked 1025 Write 8 Sent Done Done false
211722024-02-03T17:31:54.503Z 27 Acked 1026 Read 2 Sent Done Done false
211732024-02-03T17:31:54.503Z 28 Acked 1027 Write 5 Sent Done Done false
211742024-02-03T17:31:54.503Z 29 Acked 1028 Read 1 Sent Done Done false
211752024-02-03T17:31:54.503Z 30 Acked 1029 Read 2 Sent Done Done false
211762024-02-03T17:31:54.503Z STATES DS:0 DS:1 DS:2 TOTAL
211772024-02-03T17:31:54.503Z New 0 0 0 0
211782024-02-03T17:31:54.503Z Sent 27 0 0 27
211792024-02-03T17:31:54.503Z Done 3 30 30 63
211802024-02-03T17:31:54.503Z Skipped 0 0 0 0
211812024-02-03T17:31:54.503Z Error 0 0 0 0
211822024-02-03T17:31:54.503ZLast Flush: 0 1023 1023
211832024-02-03T17:31:54.503ZDownstairs last five completed:
211842024-02-03T17:31:54.503ZUpstairs last five completed: 30 29 28 27 26
211852024-02-03T17:31:54.503Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:31:53.678563139Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1919}
211862024-02-03T17:31:54.503ZWrote out file "/tmp/test_repair_verify.data"
211872024-02-03T17:31:54.514Z
211882024-02-03T17:31:54.522ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211892024-02-03T17:31:54.941ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211902024-02-03T17:31:54.941Z 0 000-019 183 196 196 334 367 367 F F F <---
211912024-02-03T17:31:54.941Z 1 020-039 187 196 196 343 367 367 F T T <---
211922024-02-03T17:31:54.941Z 2 040-059 191 191 191 355 355 355 F T T <---
211932024-02-03T17:31:54.941Z 3 060-079 194 194 194 364 364 364 F F F
211942024-02-03T17:31:54.942Z 4 080-099 195 196 196 366 367 367 F F F <---
211952024-02-03T17:31:54.942Z 5 100-119 192 196 196 359 367 367 F F F <---
211962024-02-03T17:31:54.942Z 6 120-139 192 192 192 359 359 359 F F F
211972024-02-03T17:31:54.942Z 7 140-159 189 189 189 349 349 349 F F F
211982024-02-03T17:31:54.942Z 8 160-179 189 189 189 349 349 349 F F F
211992024-02-03T17:31:54.942Z 9 180-199 194 196 196 364 367 367 F F F <---
212002024-02-03T17:31:54.942Z 10 200-219 195 195 195 366 366 366 F F F
212012024-02-03T17:31:54.942Z 11 220-239 188 188 188 345 345 345 F F F
212022024-02-03T17:31:54.942Z 12 240-259 194 194 194 362 362 362 F F F
212032024-02-03T17:31:54.942Z 13 260-279 194 196 196 364 367 367 T F F <---
212042024-02-03T17:31:54.942Z 14 280-299 195 195 195 366 366 366 F F F
212052024-02-03T17:31:54.942Z 15 300-319 194 196 196 364 367 367 F F F <---
212062024-02-03T17:31:54.943Z 16 320-339 190 196 196 351 367 367 F F F <---
212072024-02-03T17:31:54.943Z 17 340-359 190 190 190 354 354 354 F T T <---
212082024-02-03T17:31:54.943Z 18 360-379 190 196 196 354 367 367 F F F <---
212092024-02-03T17:31:54.943Z 19 380-399 192 196 196 359 367 367 T F F <---
212102024-02-03T17:31:54.943Z 20 400-419 193 193 193 361 361 361 F T T <---
212112024-02-03T17:31:54.943Z 21 420-439 190 190 190 354 354 354 F F F
212122024-02-03T17:31:54.943Z 22 440-459 188 188 188 346 346 346 F F F
212132024-02-03T17:31:54.943Z 23 460-479 194 194 194 363 363 363 F F F
212142024-02-03T17:31:54.943Z 24 480-499 190 196 196 353 367 367 T F F <---
212152024-02-03T17:31:54.943Z 25 500-519 187 196 196 343 367 367 F F F <---
212162024-02-03T17:31:54.943Z 26 520-539 184 196 196 337 367 367 F F F <---
212172024-02-03T17:31:54.943Z 27 540-559 192 192 192 357 357 357 F F F
212182024-02-03T17:31:54.943Z 28 560-579 192 196 196 360 367 367 F F F <---
212192024-02-03T17:31:54.943Z 29 580-599 193 196 196 361 367 367 F F F <---
212202024-02-03T17:31:54.943ZMax gen: 196, Max flush: 367
212212024-02-03T17:31:54.944ZError: Difference in extent metadata found!
212222024-02-03T17:31:54.944ZA Difference in extent metadata is expected here
212232024-02-03T17:31:54.944ZOn loop 98
212242024-02-03T17:31:54.944Z
212252024-02-03T17:31:54.944ZVerifying data now
212262024-02-03T17:32:01.042ZNo change after verify
212272024-02-03T17:32:01.042ZLoop: 98 Downstairs dump after verify (and repair):
212282024-02-03T17:32:01.067ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212292024-02-03T17:32:01.067Z 0 000-019 196 196 196 367 367 367 F F F
212302024-02-03T17:32:01.067Z 1 020-039 197 197 197 368 368 368 F F F
212312024-02-03T17:32:01.067Z 2 040-059 197 197 197 368 368 368 F F F
212322024-02-03T17:32:01.067Z 3 060-079 194 194 194 364 364 364 F F F
212332024-02-03T17:32:01.071Z 4 080-099 196 196 196 367 367 367 F F F
212342024-02-03T17:32:01.071Z 5 100-119 196 196 196 367 367 367 F F F
212352024-02-03T17:32:01.071Z 6 120-139 192 192 192 359 359 359 F F F
212362024-02-03T17:32:01.071Z 7 140-159 189 189 189 349 349 349 F F F
212372024-02-03T17:32:01.071Z 8 160-179 189 189 189 349 349 349 F F F
212382024-02-03T17:32:01.071Z 9 180-199 196 196 196 367 367 367 F F F
212392024-02-03T17:32:01.071Z 10 200-219 195 195 195 366 366 366 F F F
212402024-02-03T17:32:01.071Z 11 220-239 188 188 188 345 345 345 F F F
212412024-02-03T17:32:01.072Z 12 240-259 194 194 194 362 362 362 F F F
212422024-02-03T17:32:01.072Z 13 260-279 196 196 196 367 367 367 F F F
212432024-02-03T17:32:01.072Z 14 280-299 195 195 195 366 366 366 F F F
212442024-02-03T17:32:01.072Z 15 300-319 196 196 196 367 367 367 F F F
212452024-02-03T17:32:01.072Z 16 320-339 196 196 196 367 367 367 F F F
212462024-02-03T17:32:01.072Z 17 340-359 197 197 197 368 368 368 F F F
212472024-02-03T17:32:01.072Z 18 360-379 196 196 196 367 367 367 F F F
212482024-02-03T17:32:01.072Z 19 380-399 196 196 196 367 367 367 F F F
212492024-02-03T17:32:01.072Z 20 400-419 197 197 197 368 368 368 F F F
212502024-02-03T17:32:01.072Z 21 420-439 190 190 190 354 354 354 F F F
212512024-02-03T17:32:01.072Z 22 440-459 188 188 188 346 346 346 F F F
212522024-02-03T17:32:01.072Z 23 460-479 194 194 194 363 363 363 F F F
212532024-02-03T17:32:01.072Z 24 480-499 196 196 196 367 367 367 F F F
212542024-02-03T17:32:01.072Z 25 500-519 196 196 196 367 367 367 F F F
212552024-02-03T17:32:01.073Z 26 520-539 196 196 196 367 367 367 F F F
212562024-02-03T17:32:01.073Z 27 540-559 192 192 192 357 357 357 F F F
212572024-02-03T17:32:01.073Z 28 560-579 196 196 196 367 367 367 F F F
212582024-02-03T17:32:01.073Z 29 580-599 196 196 196 367 367 367 F F F
212592024-02-03T17:32:01.073ZMax gen: 197, Max flush: 368
212602024-02-03T17:32:01.077Z
212612024-02-03T17:32:01.077ZBegin loop 99
212622024-02-03T17:32:01.077ZDownstairs to restart: 1
212632024-02-03T17:32:01.108Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.287541485Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927}
212642024-02-03T17:32:01.108Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c8a96ab04e38cc0df6791339b390675d7b569c4e\",\n git_commit_timestamp: \"2024-02-03T15:19:02.000000000Z\",\n git_branch: \"renovate/base64-0.x\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288120458Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927}
212652024-02-03T17:32:01.113Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288187249Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927}
212662024-02-03T17:32:01.113ZCrucible runtime is spawned
212672024-02-03T17:32:01.113Z{"msg":"Crucible e19d2610-296f-45d6-862c-6a259d8b832a has session id: cf74849e-5d6f-4b71-b167-d160d21659ae","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288299244Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212682024-02-03T17:32:01.113Z{"msg":"Upstairs opts: Upstairs UUID: e19d2610-296f-45d6-862c-6a259d8b832a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288382183Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212692024-02-03T17:32:01.113Z{"msg":"Crucible stats registered with UUID: e19d2610-296f-45d6-862c-6a259d8b832a","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288413533Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212702024-02-03T17:32:01.113Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288472905Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927}
212712024-02-03T17:32:01.113Z{"msg":"e19d2610-296f-45d6-862c-6a259d8b832a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288526466Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212722024-02-03T17:32:01.113Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.28857964Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212732024-02-03T17:32:01.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288612335Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212742024-02-03T17:32:01.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288650565Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212752024-02-03T17:32:01.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288716689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212762024-02-03T17:32:01.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288761401Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212772024-02-03T17:32:01.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288833779Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212782024-02-03T17:32:01.114Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288878977Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212792024-02-03T17:32:01.114Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.288963422Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212802024-02-03T17:32:01.114Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.289008952Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212812024-02-03T17:32:01.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.289130525Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212822024-02-03T17:32:01.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.289247729Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212832024-02-03T17:32:01.114Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.289380986Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212842024-02-03T17:32:01.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.290403689Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212852024-02-03T17:32:01.115Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.290489038Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212862024-02-03T17:32:01.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.290538912Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212872024-02-03T17:32:01.115Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.290621573Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212882024-02-03T17:32:01.115Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.291459375Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212892024-02-03T17:32:01.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.29167497Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212902024-02-03T17:32:01.115Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.291722263Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212912024-02-03T17:32:01.120Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.291769569Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212922024-02-03T17:32:01.120Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.292795273Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212932024-02-03T17:32:01.120Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.292857622Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212942024-02-03T17:32:01.120Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.292884388Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212952024-02-03T17:32:01.120Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.292905Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212962024-02-03T17:32:01.121Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.292951391Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212972024-02-03T17:32:01.121Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293051014Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212982024-02-03T17:32:01.121Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293074741Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
212992024-02-03T17:32:01.121Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293602747Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213002024-02-03T17:32:01.121Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293673152Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213012024-02-03T17:32:01.121Z{"msg":"[0]R flush_numbers[0..12]: [367, 368, 368, 364, 367, 367, 359, 349, 349, 367, 366, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293707318Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213022024-02-03T17:32:01.121Z{"msg":"[0]R generation[0..12]: [196, 197, 197, 194, 196, 196, 192, 189, 189, 196, 195, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293732826Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213032024-02-03T17:32:01.121Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293756756Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213042024-02-03T17:32:01.121Z{"msg":"[1]R flush_numbers[0..12]: [367, 368, 368, 364, 367, 367, 359, 349, 349, 367, 366, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293800746Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213052024-02-03T17:32:01.121Z{"msg":"[1]R generation[0..12]: [196, 197, 197, 194, 196, 196, 192, 189, 189, 196, 195, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293825915Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213062024-02-03T17:32:01.121Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293849139Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213072024-02-03T17:32:01.121Z{"msg":"[2]R flush_numbers[0..12]: [367, 368, 368, 364, 367, 367, 359, 349, 349, 367, 366, 345]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293891439Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213082024-02-03T17:32:01.122Z{"msg":"[2]R generation[0..12]: [196, 197, 197, 194, 196, 196, 192, 189, 189, 196, 195, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293916368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213092024-02-03T17:32:01.122Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.29393963Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213102024-02-03T17:32:01.122Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.293984676Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213112024-02-03T17:32:01.122Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
213122024-02-03T17:32:01.122Z,"time":"2024-02-03T17:32:00.294008702Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213132024-02-03T17:32:01.122Z{"msg":"Next flush: 369","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.294157923Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213142024-02-03T17:32:01.122Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.294212174Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213152024-02-03T17:32:01.122Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.294280644Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213162024-02-03T17:32:01.122Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.294319508Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213172024-02-03T17:32:01.122Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.29438153Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213182024-02-03T17:32:01.122Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.294424277Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213192024-02-03T17:32:01.122Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.294484753Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213202024-02-03T17:32:01.123Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.29452465Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213212024-02-03T17:32:01.123Z{"msg":"e19d2610-296f-45d6-862c-6a259d8b832a is now active with session: cf74849e-5d6f-4b71-b167-d160d21659ae","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.294586258Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213222024-02-03T17:32:01.123Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.29462368Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927,"session_id":"cf74849e-5d6f-4b71-b167-d160d21659ae"}
213232024-02-03T17:32:01.123Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-03T17:32:00.294659287Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927}
213242024-02-03T17:32:01.123ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213252024-02-03T17:32:01.123ZLoading write count information from file "/tmp/test_repair_verify.data"
213262024-02-03T17:32:01.123ZRun Repair workload
213272024-02-03T17:32:01.123Z01/30 Write block 408 len 24576 data: 17 19 17 18 15 17
213282024-02-03T17:32:01.123Z02/30 Read block 170 len 40960
213292024-02-03T17:32:01.127Z03/30 Read block 5 len 28672
213302024-02-03T17:32:01.128Z04/30 Write block 335 len 12288 data: 13 12 10
213312024-02-03T17:32:01.128Z05/30 Write block 539 len 32768 data: 12 9 9 12 13 16 17 15
213322024-02-03T17:32:01.128Z06/30 Read block 304 len 36864
213332024-02-03T17:32:01.131Z07/30 Write block 523 len 40960 data: 8 8 12 15 16 15 16 14 13 14
213342024-02-03T17:32:01.131Z08/30 Read block 370 len 32768
213352024-02-03T17:32:01.131Z09/30 Write block 91 len 4096 data: 12
213362024-02-03T17:32:01.135Z10/30 Write block 281 len 16384 data: 19 21 19 18
213372024-02-03T17:32:01.135Z11/30 Write block 289 len 12288 data: 11 12 13
213382024-02-03T17:32:01.136Z12/30 Read block 479 len 36864
213392024-02-03T17:32:01.139Z13/30 Write block 318 len 36864 data: 8 16 13 14 15 14 13 11 13
213402024-02-03T17:32:01.139Z14/30 Read block 520 len 4096
213412024-02-03T17:32:01.139Z15/30 Read block 493 len 36864
213422024-02-03T17:32:01.144Z16/30 Read block 29 len 12288
213432024-02-03T17:32:01.144Z17/30 Write block 185 len 8192 data: 12 12
213442024-02-03T17:32:01.144Z18/30 Read block 338 len 20480
213452024-02-03T17:32:01.148Z19/30 Read block 109 len 8192
213462024-02-03T17:32:01.148Z20/30 Write block 537 len 4096 data: 13
213472024-02-03T17:32:01.148Z21/30 Read block 529 len 20480
213482024-02-03T17:32:01.148Z22/30 Flush
213492024-02-03T17:32:01.151Z23/30 Write block 62 len 24576 data: 16 19 19 14 14 12
213502024-02-03T17:32:01.155Z24/30 Write block 595 len 20480 data: 12 11 10 8 6
213512024-02-03T17:32:01.155Z25/30 Flush
213522024-02-03T17:32:01.159Z26/30 Flush
213532024-02-03T17:32:01.159Z27/30 Write block 143 len 28672 data: 11 12 12 8 7 8 10
213542024-02-03T17:32:01.160Z28/30 Write block 113 len 12288 data: 18 21 20
213552024-02-03T17:32:01.160Z29/30 Read block 49 len 8192
213562024-02-03T17:32:01.164Z30/30 Read block 37 len 16384
213572024-02-03T17:32:01.164Z----------------------------------------------------------------
213582024-02-03T17:32:01.164Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
213592024-02-03T17:32:01.164ZStates: Active Active Active
213602024-02-03T17:32:01.164ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213612024-02-03T17:32:01.164Z 1 Acked 1000 Write 6 Done Sent Done false
213622024-02-03T17:32:01.165Z 2 Acked 1001 Read 10 Done Sent Done false
213632024-02-03T17:32:01.165Z 3 Acked 1002 Read 7 Done Sent Done false
213642024-02-03T17:32:01.165Z 4 Acked 1003 Write 3 Done Sent Done false
213652024-02-03T17:32:01.165Z 5 Acked 1004 Write 8 Done Sent Done false
213662024-02-03T17:32:01.165Z 6 Acked 1005 Read 9 Done Sent Done false
213672024-02-03T17:32:01.165Z 7 Acked 1006 Write 10 Done Sent Done false
213682024-02-03T17:32:01.165Z 8 Acked 1007 Read 8 Done Sent Done false
213692024-02-03T17:32:01.165Z 9 Acked 1008 Write 1 Done Sent Done false
213702024-02-03T17:32:01.165Z 10 Acked 1009 Write 4 Done Sent Done false
213712024-02-03T17:32:01.165Z 11 Acked 1010 Write 3 Done Sent Done false
213722024-02-03T17:32:01.165Z 12 Acked 1011 Read 9 Done Sent Done false
213732024-02-03T17:32:01.165Z 13 Acked 1012 Write 9 Done Sent Done false
213742024-02-03T17:32:01.165Z 14 Acked 1013 Read 1 Done Sent Done false
213752024-02-03T17:32:01.166Z 15 Acked 1014 Read 9 Done Sent Done false
213762024-02-03T17:32:01.166Z 16 Acked 1015 Read 3 Done Sent Done false
213772024-02-03T17:32:01.166Z 17 Acked 1016 Write 2 Done Sent Done false
213782024-02-03T17:32:01.166Z 18 Acked 1017 Read 5 Done Sent Done false
213792024-02-03T17:32:01.166Z 19 Acked 1018 Read 2 Done Sent Done false
213802024-02-03T17:32:01.166Z 20 Acked 1019 Write 1 Done Sent Done false
213812024-02-03T17:32:01.166Z 21 Acked 1020 Read 5 Done Sent Done false
213822024-02-03T17:32:01.166Z 22 Acked 1021 Flush 0 Done Sent Done false
213832024-02-03T17:32:01.166Z 23 Acked 1022 Write 6 Done Sent Done false
213842024-02-03T17:32:01.166Z 24 Acked 1023 Write 5 Done Sent Done false
213852024-02-03T17:32:01.166Z 25 Acked 1024 Flush 0 Done Sent Done false
213862024-02-03T17:32:01.166Z 26 Acked 1025 Flush 0 Done Sent Done false
213872024-02-03T17:32:01.167Z 27 Acked 1026 Write 7 Done Sent Done false
213882024-02-03T17:32:01.167Z 28 Acked 1027 Write 3 Done Sent Done false
213892024-02-03T17:32:01.167Z 29 Acked 1028 Read 2 Done Sent Done false
213902024-02-03T17:32:01.167Z 30 Acked 1029 Read 4 Done Sent Done false
213912024-02-03T17:32:01.167Z STATES DS:0 DS:1 DS:2 TOTAL
213922024-02-03T17:32:01.167Z New 0 0 0 0
213932024-02-03T17:32:01.167Z Sent 0 30 0 30
213942024-02-03T17:32:01.167Z Done 30 0 30 60
213952024-02-03T17:32:01.167Z Skipped 0 0 0 0
213962024-02-03T17:32:01.167Z Error 0 0 0 0
213972024-02-03T17:32:01.167ZLast Flush: 1025 0 1025
213982024-02-03T17:32:01.167ZDownstairs last five completed:
213992024-02-03T17:32:01.167ZUpstairs last five completed: 30 29 28 27 26
214002024-02-03T17:32:01.168Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-03T17:32:00.341359434Z","hostname":"ip-10-150-1-93.us-west-2.compute.internal","pid":1927}
214012024-02-03T17:32:01.168ZWrote out file "/tmp/test_repair_verify.data"
214022024-02-03T17:32:01.172Z
214032024-02-03T17:32:01.177ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
214042024-02-03T17:32:01.206ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214052024-02-03T17:32:01.206Z 0 000-019 196 196 196 367 367 367 F F F
214062024-02-03T17:32:01.207Z 1 020-039 197 197 197 368 368 368 F F F
214072024-02-03T17:32:01.207Z 2 040-059 197 197 197 368 368 368 F F F
214082024-02-03T17:32:01.207Z 3 060-079 198 194 198 370 364 370 F F F <---
214092024-02-03T17:32:01.207Z 4 080-099 198 196 198 369 367 369 F F F <---
214102024-02-03T17:32:01.207Z 5 100-119 196 196 196 367 367 367 T F T <---
214112024-02-03T17:32:01.207Z 6 120-139 192 192 192 359 359 359 F F F
214122024-02-03T17:32:01.218Z 7 140-159 189 189 189 349 349 349 T F T <---
214132024-02-03T17:32:01.218Z 8 160-179 189 189 189 349 349 349 F F F
214142024-02-03T17:32:01.218Z 9 180-199 198 196 198 369 367 369 F F F <---
214152024-02-03T17:32:01.219Z 10 200-219 195 195 195 366 366 366 F F F
214162024-02-03T17:32:01.219Z 11 220-239 188 188 188 345 345 345 F F F
214172024-02-03T17:32:01.219Z 12 240-259 194 194 194 362 362 362 F F F
214182024-02-03T17:32:01.219Z 13 260-279 196 196 196 367 367 367 F F F
214192024-02-03T17:32:01.219Z 14 280-299 198 195 198 369 366 369 F F F <---
214202024-02-03T17:32:01.219Z 15 300-319 198 196 198 369 367 369 F F F <---
214212024-02-03T17:32:01.220Z 16 320-339 198 196 198 369 367 369 F F F <---
214222024-02-03T17:32:01.220Z 17 340-359 197 197 197 368 368 368 F F F
214232024-02-03T17:32:01.220Z 18 360-379 196 196 196 367 367 367 F F F
214242024-02-03T17:32:01.220Z 19 380-399 196 196 196 367 367 367 F F F
214252024-02-03T17:32:01.220Z 20 400-419 198 197 198 369 368 369 F F F <---
214262024-02-03T17:32:01.220Z 21 420-439 190 190 190 354 354 354 F F F
214272024-02-03T17:32:01.220Z 22 440-459 188 188 188 346 346 346 F F F
214282024-02-03T17:32:01.221Z 23 460-479 194 194 194 363 363 363 F F F
214292024-02-03T17:32:01.221Z 24 480-499 196 196 196 367 367 367 F F F
214302024-02-03T17:32:01.221Z 25 500-519 196 196 196 367 367 367 F F F
214312024-02-03T17:32:01.221Z 26 520-539 198 196 198 369 367 369 F F F <---
214322024-02-03T17:32:01.221Z 27 540-559 198 192 198 369 357 369 F F F <---
214332024-02-03T17:32:01.221Z 28 560-579 196 196 196 367 367 367 F F F
214342024-02-03T17:32:01.221Z 29 580-599 198 196 198 370 367 370 F F F <---
214352024-02-03T17:32:01.221ZMax gen: 198, Max flush: 370
214362024-02-03T17:32:01.221ZError: Difference in extent metadata found!
214372024-02-03T17:32:01.221ZA Difference in extent metadata is expected here
214382024-02-03T17:32:01.221ZOn loop 99
214392024-02-03T17:32:01.221Z
214402024-02-03T17:32:01.222ZVerifying data now
214412024-02-03T17:32:06.281ZNo change after verify
214422024-02-03T17:32:06.281ZLoop: 99 Downstairs dump after verify (and repair):
214432024-02-03T17:32:06.305ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214442024-02-03T17:32:06.305Z 0 000-019 196 196 196 367 367 367 F F F
214452024-02-03T17:32:06.308Z 1 020-039 197 197 197 368 368 368 F F F
214462024-02-03T17:32:06.309Z 2 040-059 197 197 197 368 368 368 F F F
214472024-02-03T17:32:06.309Z 3 060-079 198 198 198 370 370 370 F F F
214482024-02-03T17:32:06.309Z 4 080-099 198 198 198 369 369 369 F F F
214492024-02-03T17:32:06.309Z 5 100-119 199 199 199 371 371 371 F F F
214502024-02-03T17:32:06.309Z 6 120-139 192 192 192 359 359 359 F F F
214512024-02-03T17:32:06.309Z 7 140-159 199 199 199 371 371 371 F F F
214522024-02-03T17:32:06.309Z 8 160-179 189 189 189 349 349 349 F F F
214532024-02-03T17:32:06.309Z 9 180-199 198 198 198 369 369 369 F F F
214542024-02-03T17:32:06.309Z 10 200-219 195 195 195 366 366 366 F F F
214552024-02-03T17:32:06.309Z 11 220-239 188 188 188 345 345 345 F F F
214562024-02-03T17:32:06.309Z 12 240-259 194 194 194 362 362 362 F F F
214572024-02-03T17:32:06.309Z 13 260-279 196 196 196 367 367 367 F F F
214582024-02-03T17:32:06.309Z 14 280-299 198 198 198 369 369 369 F F F
214592024-02-03T17:32:06.309Z 15 300-319 198 198 198 369 369 369 F F F
214602024-02-03T17:32:06.309Z 16 320-339 198 198 198 369 369 369 F F F
214612024-02-03T17:32:06.309Z 17 340-359 197 197 197 368 368 368 F F F
214622024-02-03T17:32:06.309Z 18 360-379 196 196 196 367 367 367 F F F
214632024-02-03T17:32:06.309Z 19 380-399 196 196 196 367 367 367 F F F
214642024-02-03T17:32:06.309Z 20 400-419 198 198 198 369 369 369 F F F
214652024-02-03T17:32:06.309Z 21 420-439 190 190 190 354 354 354 F F F
214662024-02-03T17:32:06.309Z 22 440-459 188 188 188 346 346 346 F F F
214672024-02-03T17:32:06.309Z 23 460-479 194 194 194 363 363 363 F F F
214682024-02-03T17:32:06.309Z 24 480-499 196 196 196 367 367 367 F F F
214692024-02-03T17:32:06.310Z 25 500-519 196 196 196 367 367 367 F F F
214702024-02-03T17:32:06.310Z 26 520-539 198 198 198 369 369 369 F F F
214712024-02-03T17:32:06.310Z 27 540-559 198 198 198 369 369 369 F F F
214722024-02-03T17:32:06.310Z 28 560-579 196 196 196 367 367 367 F F F
214732024-02-03T17:32:06.310Z 29 580-599 198 198 198 370 370 370 F F F
214742024-02-03T17:32:06.310ZMax gen: 199, Max flush: 371
214752024-02-03T17:32:06.313Z9:32 Test duration
214762024-02-03T17:32:06.313ZTest completed
214772024-02-03T17:32:06.313Z
214782024-02-03T17:32:06.313Zreal 9:31.883613332
214792024-02-03T17:32:06.313Zuser 2:49.051208607
214802024-02-03T17:32:06.313Zsys 29.933272529
214812024-02-03T17:32:06.314Ztrap 0.143750772
214822024-02-03T17:32:06.314Ztflt 0.093106068
214832024-02-03T17:32:06.314Zdflt 0.118328607
214842024-02-03T17:32:06.314Zkflt 0.050468584
214852024-02-03T17:32:06.314Zlock 7:31:50.019941023
214862024-02-03T17:32:06.314Zslp 44:49.114239944
214872024-02-03T17:32:06.314Zlat 10.262977992
214882024-02-03T17:32:06.314Zstop 0.067543519
214892024-02-03T17:32:06.317Zprocess exited: duration 585865 ms, exit code 0
 
214902024-02-03T17:32:06.324Zfound 4 output files
214912024-02-03T17:32:06.324Zuploading: /tmp/test_repair_ds8810.txt (45628 bytes)
214922024-02-03T17:32:07.343Zuploaded: /tmp/test_repair_ds8810.txt
214932024-02-03T17:32:07.343Zuploading: /tmp/test_repair_ds8820.txt (19589 bytes)
214942024-02-03T17:32:08.355Zuploaded: /tmp/test_repair_ds8820.txt
214952024-02-03T17:32:08.356Zuploading: /tmp/test_repair_ds8830.txt (44974 bytes)
214962024-02-03T17:32:09.368Zuploaded: /tmp/test_repair_ds8830.txt
214972024-02-03T17:32:09.369Zuploading: /tmp/test_repair_out.txt (82299 bytes)
214982024-02-03T17:32:10.385Zuploaded: /tmp/test_repair_out.txt